Information Retrieval - Image Event Detection - Paolo Cortis¶
Running experiments with different pipelines and machine learning models for a multi class image event classification task on soccer images.
Libraries¶
In [1]:
import os
import cv2
from skimage import transform
from tensorflow.python.ops.numpy_ops import np_config
np_config.enable_numpy_behavior()
import random
import pandas as pd
import numpy as np
from tqdm import tqdm
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import activations, layers, optimizers, losses
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras import Sequential
from extra_keras_metrics import get_standard_binary_metrics, get_minimal_multiclass_metrics
from sklearn.metrics import accuracy_score, average_precision_score, roc_auc_score
from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
from sklearn.metrics import ConfusionMatrixDisplay
import seaborn as sns
from matplotlib import pyplot as plt
%matplotlib inline
In [2]:
soccer_images_dim = 80
Data Retrieval¶
Import of the main datasets used:
For model training and evaluation
- Soccer Events: images of the soccer events to classify.
- Generic Events: images of generic soccer scenes (not events) and random images.
For further model evaluation, more diverse soccer and random images
- Real Soccer: few images of the soccer events (not from the football analysis dataset).
- Test dataset: few soccer and random images (not from soccer and generic events dataset).
In [3]:
def loadData(root, rgb):
""" Return images from the dataset with the corresponding label from a root directory.
Parameters
------------------------
root: str
The root folder with dataset.
rgb: bool
Specify whether to import images in rgb or gray scale.
Returns
------------------------
data: dict
A dictionary with data images.
folder_labels: list
A list with the images labels."""
data = {}
folder_labels = []
for folder_name in os.listdir(root):
folder_path = os.path.join(root, folder_name)
# Check if the item is a directory
if os.path.isdir(folder_path):
folder_files = []
for file_name in os.listdir(folder_path):
img = cv2.imread(os.path.join(folder_path, file_name))
if img is not None:
resizedImg = cv2.resize(img, (soccer_images_dim,soccer_images_dim), interpolation=cv2.INTER_CUBIC)
RGBImg = cv2.cvtColor(resizedImg, cv2.COLOR_BGR2RGB)
if rgb == False:
grayImg = cv2.cvtColor(RGBImg, cv2.COLOR_RGB2GRAY)
folder_files.append(grayImg)
else:
folder_files.append(RGBImg)
folder_labels.append(folder_name)
data[folder_name] = folder_files
return data, folder_labels
In [10]:
# Soccer events
folderSoccer = r'D:\MachineLearning\Datasets\Soccer\TrainTestSoccer'
# Generic soccer and random events
folderGenericEvents = r'D:\MachineLearning\Datasets\Soccer\GenericEvents'
# Broadly different soccer events images)
folderRSoccer = r'D:\MachineLearning\Datasets\Soccer\RealSoccer'
# RealSoccer + events and genericSoccer images
folderMyTestDataset = r'D:\MachineLearning\Datasets\Soccer\myDataset'
data_soccer, labels_soccer = loadData(folderSoccer, False)
data_soccer_rgb, _ = loadData(folderSoccer, True)
data_generic_events, labels_generic_events = loadData(folderGenericEvents, False)
real_data_soccer, real_labels_soccer = loadData(folderRSoccer, False)
data_myTest, labels_myTest = loadData(folderMyTestDataset, False)
In [14]:
# Save datasets
import pickle
# Save Dataset (save dictionary to data_soccer.pkl file)
with open('data/data_soccer.pkl', 'wb') as fp:
pickle.dump(data_soccer, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to data_soccer_rgb.pkl file)
with open('data/data_soccer_rgb.pkl', 'wb') as fp:
pickle.dump(data_soccer_rgb, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to labels_soccer.pkl file)
with open('data/labels_soccer.pkl', 'wb') as fp:
pickle.dump(labels_soccer, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to data_generic_events.pkl file)
with open('data/data_generic_events.pkl', 'wb') as fp:
pickle.dump(data_generic_events, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to labels_generic_events.pkl file)
with open('data/labels_generic_events.pkl', 'wb') as fp:
pickle.dump(labels_generic_events, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to real_data_soccer.pkl file)
with open('data/real_data_soccer.pkl', 'wb') as fp:
pickle.dump(real_data_soccer, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to real_labels_soccer.pkl file)
with open('data/real_labels_soccer.pkl', 'wb') as fp:
pickle.dump(real_labels_soccer, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to data_myTest.pkl file)
with open('data/data_myTest.pkl', 'wb') as fp:
pickle.dump(data_myTest, fp)
print('dictionary saved successfully to file')
# Save Dataset (save dictionary to labels_myTest.pkl file)
with open('data/labels_myTest.pkl', 'wb') as fp:
pickle.dump(labels_myTest, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file dictionary saved successfully to file
In [4]:
# Load datasets
import pickle
# Load Dataset (Read dictionary pkl file)
with open('data/data_soccer.pkl', 'rb') as fp:
data_soccer = pickle.load(fp)
# Load Dataset (Read dictionary pkl file)
with open('data/data_soccer_rgb.pkl', 'rb') as fp:
data_soccer_rgb = pickle.load(fp)
with open('data/labels_soccer.pkl', 'rb') as fp:
labels_soccer = pickle.load(fp)
with open('data/data_generic_events.pkl', 'rb') as fp:
data_generic_events = pickle.load(fp)
with open('data/labels_generic_events.pkl', 'rb') as fp:
labels_generic_events = pickle.load(fp)
with open('data/real_data_soccer.pkl', 'rb') as fp:
real_data_soccer = pickle.load(fp)
with open('data/real_labels_soccer.pkl', 'rb') as fp:
real_labels_soccer = pickle.load(fp)
with open('data/data_myTest.pkl', 'rb') as fp:
data_myTest = pickle.load(fp)
with open('data/labels_myTest.pkl', 'rb') as fp:
labels_myTest = pickle.load(fp)
Data preprocessing and visualization¶
In [5]:
print("-- Soccer Events and real soccer events Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_soccer):
print(f"--- {folder} ---")
print(f"Soccer shape: {len(data_soccer[folder])} {data_soccer[folder][1].shape}")
print(f"Real soccer shape: {len(real_data_soccer[folder])} {real_data_soccer[folder][1].shape}")
soccer_sample = data_soccer['Cards'][22]
real_soccer_sample = real_data_soccer['Center'][1]
print("-"*30)
print(f"Soccer label: {labels_soccer[22]}")
print(f"Real soccer label: {real_labels_soccer[11]}")
plt.subplot(1,2,1),plt.imshow(soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(real_soccer_sample, cmap='gray', vmin=0, vmax=255)
-- Soccer Events and real soccer events Dataset -- [n.images, (width, height, depth)] --- Cards --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Center --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Corner --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Free-Kick --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Left --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Penalty --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Right --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- Tackle --- Soccer shape: 6000 (80, 80) Real soccer shape: 10 (80, 80) --- To Substitue --- Soccer shape: 5999 (80, 80) Real soccer shape: 10 (80, 80) ------------------------------ Soccer label: Cards Real soccer label: Center
Out[5]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246e3f8feb0>)
In [6]:
print("-- Generic Events Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_generic_events):
print(f"--- {folder} ---")
print(f"Soccer shape: {len(data_generic_events[folder])} {data_generic_events[folder][1].shape}")
generic_soccer_sample = data_generic_events['Soccer'][20]
random_image_sample = data_generic_events['Random'][20]
print("-"*30)
print(f"Label: {labels_generic_events[1420]}")
print(f"Label: {labels_generic_events[20]}")
plt.subplot(1,2,1),plt.imshow(generic_soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(random_image_sample, cmap='gray', vmin=0, vmax=255)
-- Generic Events Dataset -- [n.images, (width, height, depth)] --- Random --- Soccer shape: 1400 (80, 80) --- Soccer --- Soccer shape: 1200 (80, 80) ------------------------------ Label: Soccer Label: Random
Out[6]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246e60dafd0>)
In [7]:
print("-- myTest Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_myTest):
print(f"--- {folder} ---")
print(f"Soccer shape: {len(data_myTest[folder])} {data_myTest[folder][1].shape}")
generic_soccer_sample = data_myTest['Cards'][6]
random_image_sample = data_myTest['Events'][6]
print("-"*30)
print(f"Label: {labels_myTest[6]}")
print(f"Label: {labels_myTest[36]}")
plt.subplot(1,2,1),plt.imshow(generic_soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(random_image_sample, cmap='gray', vmin=0, vmax=255)
-- myTest Dataset -- [n.images, (width, height, depth)] --- Cards --- Soccer shape: 10 (80, 80) --- Center --- Soccer shape: 10 (80, 80) --- Corner --- Soccer shape: 10 (80, 80) --- Events --- Soccer shape: 10 (80, 80) --- Free-Kick --- Soccer shape: 10 (80, 80) --- Generic --- Soccer shape: 10 (80, 80) --- Left --- Soccer shape: 10 (80, 80) --- Penalty --- Soccer shape: 10 (80, 80) --- Right --- Soccer shape: 10 (80, 80) --- Tackle --- Soccer shape: 10 (80, 80) --- To Substitue --- Soccer shape: 10 (80, 80) ------------------------------ Label: Cards Label: Events
Out[7]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246ee822f70>)
Label encoding¶
In [8]:
def one_hot_encoding(
labels: pd.DataFrame,
num_classes: int
) -> pd.DataFrame:
""" Return the labels encoded with one-hot encoding.
Parameters
------------------------
labels: pd.DataFrame
The labels to be one-hot encoded.
num_classes: int
The number of classes.
Returns
------------------------
pd.DataFrame: pd.DataFrame
The one-hot encoded labels."""
return pd.DataFrame(to_categorical(labels, num_classes))
In [9]:
n_generic_soccer_samples = labels_generic_events.count("Soccer")
In [10]:
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
print("Soccer Events Labels")
labels_soccer = pd.DataFrame(le.fit_transform(labels_soccer))
labels_soccer = one_hot_encoding(labels_soccer, 9)
print(labels_soccer.sample(3))
print("-"*30)
print("Real Soccer Events Labels")
real_labels_soccer = pd.DataFrame(le.fit_transform(real_labels_soccer))
real_labels_soccer = one_hot_encoding(real_labels_soccer, 9)
print(real_labels_soccer.sample(3))
print("-"*30)
print("Soccer Events + Generic Labels")
labels_soccer_with_generic = labels_soccer.copy()
labels_soccer_with_generic[9] = 0.0
new_rows = pd.DataFrame([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]] * n_generic_soccer_samples)
labels_soccer_with_generic = pd.concat([labels_soccer_with_generic, new_rows])
print(labels_soccer_with_generic.sample(3))
print("-"*30)
print("Generic Events Labels")
labels_generic_events = pd.DataFrame(le.fit_transform(labels_generic_events))
labels_generic_events = one_hot_encoding(labels_generic_events, 2)
print(labels_generic_events.sample(3))
print("-"*30)
print("my Test Labels")
labels_myTest = pd.DataFrame(le.fit_transform(labels_myTest))
labels_myTest = one_hot_encoding(labels_myTest, 11)
print(labels_myTest.sample(3))
Soccer Events Labels
0 1 2 3 4 5 6 7 8
22536 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
13976 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
43885 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
------------------------------
Real Soccer Events Labels
0 1 2 3 4 5 6 7 8
71 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
28 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
48 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0
------------------------------
Soccer Events + Generic Labels
0 1 2 3 4 5 6 7 8 9
12044 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
46546 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0
7338 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
------------------------------
Generic Events Labels
0 1
1347 1.0 0.0
2438 0.0 1.0
2381 0.0 1.0
------------------------------
my Test Labels
0 1 2 3 4 5 6 7 8 9 10
49 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0
94 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
71 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
Data Augmentation¶
Function to perform data augmentation to expand the dataset size and improve generalization capabilities.
In [11]:
def augment_image(input_image):
""" Return the augmented input image (randomly applies rotation, shear, zoom, crop, flip, brightness adjust)
Parameters
------------------------
input_image: np.array
The input image.
Returns
------------------------
np.array: np.array
The augmented image."""
# Randomly rotate the image (between -30 and 30 degrees)
height, width = input_image.shape[:2]
angle = random.randint(-30, 30)
rotation_matrix = cv2.getRotationMatrix2D((width / 2, height / 2), angle, 1)
input_image = cv2.warpAffine(input_image, rotation_matrix, (width, height))
# Apply shearing (0.15)
shear_value = random.uniform(-0.15, 0.15)
shear_matrix = np.array([[1.0, shear_value, 0.0], [0.0, 1.0, 0.0]])
input_image = cv2.warpAffine(input_image, shear_matrix, (width, height))
input_image = np.expand_dims(input_image, axis=-1)
# Randomly zoom (scaling factor between 0.9 and 1.1)
zoom_factor = tf.random.uniform(shape=[], minval=0.9, maxval=1.1)
new_size = tf.cast(tf.shape(input_image)[:2], tf.float32) * zoom_factor
input_image = tf.image.resize(input_image, tf.cast(new_size, tf.int32))
# Randomly crop (90% of the original size)
crop_size = tf.cast(tf.shape(input_image)[:2] * 0.9, tf.int32)
input_image = tf.image.random_crop(input_image, size=[crop_size[0], crop_size[1], 1])
# Randomly adjust brightness (by 0.2)
input_image = tf.image.random_brightness(input_image, max_delta=0.2)
# Reshape the augmented image
input_image = np.array(input_image)
input_image = cv2.resize(input_image, (soccer_images_dim,soccer_images_dim), interpolation=cv2.INTER_CUBIC)
input_image = input_image.reshape(soccer_images_dim, soccer_images_dim, 1)
return input_image
In [12]:
print("-- Just for explanatory purpose --")
# Augment the image
input_image = np.expand_dims(soccer_sample, axis=-1)
augmented_image = augment_image(input_image)
# Display the original and augmented images
plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.title('Original Image')
plt.imshow(soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.axis('off')
plt.subplot(1,2,2)
plt.title('Augmented Image')
plt.imshow(augmented_image, cmap='gray', vmin=0, vmax=255)
plt.axis('off')
plt.show()
-- Just for explanatory purpose --
Learning¶
The technique used to generate the holdouts for the evaluation of the various architectures is the stratified Monte-Carlo method, producing each time a different arrangement of the training and testing set, while keeping roughly the same class balance.
2 holdouts are considered to train and evaluate each model, 20% of the whole dataset is used for testing, 80% for training.
Below are also some functions to perform the training and evaluation of models.
In [13]:
number_of_splits = 2
holdouts_generator = StratifiedShuffleSplit(
n_splits = number_of_splits,
test_size = 0.2,
random_state = 42
)
In [14]:
def train_model(
model: tf.keras.Model,
x_train: np.ndarray,
x_test: np.ndarray,
y_train: np.ndarray,
y_test: np.ndarray,
epochs: int,
batch_size: int
):
""" Train the model and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
Parameters
------------------------
model: tf.keras.Model
The model to train.
x_train: np.ndarray
The input data for training.
x_test: np.ndarray
The input data for testing.
y_train: np.ndarray
The labels of the input data for training.
y_test: np.ndarray
The labels of the input data for testing.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
The performance metrics and history of the model."""
model_history = model.fit(
x_train, y_train,
validation_data = (x_test, y_test),
epochs = epochs,
batch_size = batch_size,
callbacks = [EarlyStopping("val_loss", patience=2)]
)
train_evaluation = model.evaluate(x_train, y_train, return_dict=True)
test_evaluation = model.evaluate(x_test, y_test, return_dict=True)
metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
return metrics, model_history
In [15]:
def train_model_cp(
model: tf.keras.Model,
x_train: np.ndarray,
x_test: np.ndarray,
y_train: np.ndarray,
y_test: np.ndarray,
epochs: int,
batch_size: int,
cp: keras.callbacks.ModelCheckpoint
):
""" Train the model with checkpoints and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
Parameters
------------------------
model: tf.keras.Model
The model to train.
x_train: np.ndarray
The input data for training.
x_test: np.ndarray
The input data for testing.
y_train: np.ndarray
The labels of the input data for training.
y_test: np.ndarray
The labels of the input data for testing.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
cp: keras.callbacks.ModelCheckpoint
Checkpoint callback.
Returns
------------------------
The performance metrics and history of the model."""
model_history = model.fit(
x_train, y_train,
validation_data = (x_test, y_test),
epochs = epochs,
batch_size = batch_size,
callbacks = [EarlyStopping("val_loss", patience=4), cp]
)
#train_evaluation = model.evaluate(x_train, y_train, return_dict=True)
test_evaluation = model.evaluate(x_test, y_test, return_dict=True)
#metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
metrics = {"test_evaluation": test_evaluation}
return metrics, model_history
In [16]:
def get_predictions(
model: tf.keras.Model,
x_test: np.ndarray,
batch_size: int
):
""" Return the predictions of the model over the test data.
Parameters
------------------------
model: tf.keras.Model
The model to train.
x_test: np.ndarray
The input data for testing.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
The predictions of the model over the test data."""
test_predictions = model.predict(
x_test,
batch_size = batch_size
)
return test_predictions
Evaluate a model¶
Next are a set of functions to evaluate and visualize the performances of models.
To have a statistically sound estimate of an architecture performance, multiple models are built and trained, each with the same architecture, over different portions of the data (holdouts) and, the average performance of those, is considered as an estimate of the overall performance of the architecture.
In [17]:
def model_metrics_holdout_estimate(
model_metrics: list,
holdout_total_number: int
) -> dict[str, float]:
""" Return the average metrics (accuracy, AUROC, AUPRC) of the model accross all the holdouts for train and test.
Parameters
------------------------
model_metrics: list
The list with the train/test metrics (accuracy, AUROC, AUPRC) for all the holdouts.
holdout_total_number: int
The number of total holdouts.
Returns
------------------------
dict: dict[str, float]
The average metrics of the model accross all the holdouts for train and test."""
total_accuracy_train = 0
total_accuracy_test = 0
total_AUROC_train = 0
total_AUROC_test = 0
total_AUPRC_train = 0
total_AUPRC_test = 0
for holdout in model_metrics:
total_accuracy_train += holdout["train_evaluation"]['accuracy']
total_AUROC_train += holdout["train_evaluation"]['AUROC']
total_AUPRC_train += holdout["train_evaluation"]['AUPRC']
for holdout in model_metrics:
total_accuracy_test += holdout["test_evaluation"]['accuracy']
total_AUROC_test += holdout["test_evaluation"]['AUROC']
total_AUPRC_test += holdout["test_evaluation"]['AUPRC']
avg_accuracy_train = total_accuracy_train / holdout_total_number
avg_accuracy_test = total_accuracy_test / holdout_total_number
avg_AUROC_train = total_AUROC_train / holdout_total_number
avg_AUROC_test = total_AUROC_test / holdout_total_number
avg_AUPRC_train = total_AUPRC_train / holdout_total_number
avg_AUPRC_test = total_AUPRC_test / holdout_total_number
return {
'accuracy_train': avg_accuracy_train,
'accuracy_test': avg_accuracy_test,
'AUROC_train': avg_AUROC_train,
'AUROC_test': avg_AUROC_test,
'AUPRC_train': avg_AUPRC_train,
'AUPRC_test': avg_AUPRC_test,
}
In [18]:
def plot_train_history(model_history):
""" Plots the metrics (accuracy, AUROC, AUPRC) of the model across the epochs and for each holdout.
Parameters
------------------------
model_history: history
The training record of the model."""
for i in range(len(model_history)):
fig = plt.figure(figsize=(12.7,4.51))
fig.suptitle(str(i+1) + " Holdout")
# Accuracy
plt.subplot(1,3,1)
plt.plot(model_history[i].history['accuracy'])
plt.plot(model_history[i].history['val_accuracy'])
plt.title('Accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
# AUROC
plt.subplot(1,3,2)
plt.plot(model_history[i].history['AUROC'])
plt.plot(model_history[i].history['val_AUROC'])
plt.title('AUROC')
plt.ylabel('AUROC')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
# AUPRC
plt.subplot(1,3,3)
plt.plot(model_history[i].history['AUPRC'])
plt.plot(model_history[i].history['val_AUPRC'])
plt.title('AUPRC')
plt.ylabel('AUPRC')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.subplots_adjust(wspace=0.3)
plt.show()
CNN¶
A convolutional neural network consists a set of convolutional layers (convolution+maxPooling) and a set of dense layer. To avoid overfitting of the model dropout layers were added in the dense portion.
Here is the function to build the main model.
In [19]:
def build_soccer_CNN(
input_shape: tuple
) -> tf.keras.Model:
""" Returns the basic soccer CNN model.
Parameters
------------------------
input_shape: tuple
The shape of the input.
Returns
------------------------
model: model
The soccer model."""
CNN = Sequential(name="Soccer_CNN")
CNN.add(layers.Input(input_shape))
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Flatten())
for i in range(2):
CNN.add(layers.Dense(units = 128, activation='relu'))
CNN.add(layers.Dropout(0.5))
CNN.add(layers.Dense(9, activation='softmax'))
CNN.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=get_minimal_multiclass_metrics()
)
CNN.summary()
return CNN
In [20]:
#data_soccer is a dict containing ['events'...] so put together classes to evaluate
data_soccer_all = []
for event in data_soccer:
for element in data_soccer[event]:
data_soccer_all.append(element)
print(f"Data shape: {np.array(data_soccer_all).shape}")
print(f"Labels shape: {labels_soccer.shape}")
Data shape: (53999, 80, 80) Labels shape: (53999, 9)
In [21]:
data_soccer_all = [np.expand_dims(x, axis=-1) for x in data_soccer_all]
In [22]:
epochs = 100
batch_size = 14
Main training loop - Soccer CNN¶
- Normalize images' values in 0-1 range.
- Build and train the model over the training set and test it over the test set for the different holdouts.
In [23]:
print("---- Soccer CNN ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_all, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([data_soccer_all[x]/255 for x in train_indices]), np.array([data_soccer_all[x]/255 for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
# Build CNN
CNN = build_soccer_CNN(x_train[0].shape)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_model(
CNN,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size
)
predictions = get_predictions(CNN, x_test, batch_size=batch_size)
CNN_holdout_predictions.append(predictions)
CNN_holdout_test_labels.append(y_test)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(CNN)
---- Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d (MaxPooling2D (None, 40, 40, 128) 0
)
conv2d_1 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_1 (MaxPooling (None, 20, 20, 128) 0
2D)
conv2d_2 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_2 (MaxPooling (None, 10, 10, 256) 0
2D)
conv2d_3 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_3 (MaxPooling (None, 5, 5, 256) 0
2D)
flatten (Flatten) (None, 6400) 0
dense (Dense) (None, 128) 819328
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 128) 16512
dropout_1 (Dropout) (None, 128) 0
dense_2 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
3086/3086 [==============================] - 37s 10ms/step - loss: 1.2341 - accuracy: 0.5270 - recall: 0.3408 - precision: 0.7308 - AUROC: 0.9027 - AUPRC: 0.5983 - val_loss: 0.6373 - val_accuracy: 0.7908 - val_recall: 0.6710 - val_precision: 0.8773 - val_AUROC: 0.9761 - val_AUPRC: 0.8686
Epoch 2/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.7161 - accuracy: 0.7545 - recall: 0.6664 - precision: 0.8304 - AUROC: 0.9665 - AUPRC: 0.8345 - val_loss: 0.4194 - val_accuracy: 0.8614 - val_recall: 0.8125 - val_precision: 0.9001 - val_AUROC: 0.9884 - val_AUPRC: 0.9339
Epoch 3/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.5331 - accuracy: 0.8254 - recall: 0.7739 - precision: 0.8727 - AUROC: 0.9799 - AUPRC: 0.8999 - val_loss: 0.3611 - val_accuracy: 0.8840 - val_recall: 0.8471 - val_precision: 0.9158 - val_AUROC: 0.9907 - val_AUPRC: 0.9476
Epoch 4/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.4401 - accuracy: 0.8571 - recall: 0.8192 - precision: 0.8926 - AUROC: 0.9856 - AUPRC: 0.9282 - val_loss: 0.3377 - val_accuracy: 0.8894 - val_recall: 0.8622 - val_precision: 0.9161 - val_AUROC: 0.9913 - val_AUPRC: 0.9519
Epoch 5/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3766 - accuracy: 0.8792 - recall: 0.8504 - precision: 0.9079 - AUROC: 0.9884 - AUPRC: 0.9447 - val_loss: 0.3129 - val_accuracy: 0.9000 - val_recall: 0.8756 - val_precision: 0.9228 - val_AUROC: 0.9920 - val_AUPRC: 0.9592
Epoch 6/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3438 - accuracy: 0.8902 - recall: 0.8657 - precision: 0.9179 - AUROC: 0.9902 - AUPRC: 0.9531 - val_loss: 0.2792 - val_accuracy: 0.9126 - val_recall: 0.8922 - val_precision: 0.9308 - val_AUROC: 0.9934 - val_AUPRC: 0.9663
Epoch 7/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3024 - accuracy: 0.9041 - recall: 0.8832 - precision: 0.9254 - AUROC: 0.9920 - AUPRC: 0.9622 - val_loss: 0.2689 - val_accuracy: 0.9124 - val_recall: 0.8954 - val_precision: 0.9322 - val_AUROC: 0.9936 - val_AUPRC: 0.9680
Epoch 8/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2789 - accuracy: 0.9118 - recall: 0.8935 - precision: 0.9313 - AUROC: 0.9929 - AUPRC: 0.9669 - val_loss: 0.2710 - val_accuracy: 0.9140 - val_recall: 0.8984 - val_precision: 0.9311 - val_AUROC: 0.9935 - val_AUPRC: 0.9687
Epoch 9/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2614 - accuracy: 0.9195 - recall: 0.9030 - precision: 0.9377 - AUROC: 0.9934 - AUPRC: 0.9701 - val_loss: 0.2289 - val_accuracy: 0.9295 - val_recall: 0.9116 - val_precision: 0.9459 - val_AUROC: 0.9949 - val_AUPRC: 0.9760
Epoch 10/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2399 - accuracy: 0.9256 - recall: 0.9103 - precision: 0.9429 - AUROC: 0.9941 - AUPRC: 0.9738 - val_loss: 0.2399 - val_accuracy: 0.9244 - val_recall: 0.9094 - val_precision: 0.9385 - val_AUROC: 0.9943 - val_AUPRC: 0.9741
Epoch 11/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2241 - accuracy: 0.9293 - recall: 0.9150 - precision: 0.9458 - AUROC: 0.9948 - AUPRC: 0.9768 - val_loss: 0.2313 - val_accuracy: 0.9267 - val_recall: 0.9128 - val_precision: 0.9420 - val_AUROC: 0.9947 - val_AUPRC: 0.9759
1350/1350 [==============================] - 8s 6ms/step - loss: 0.1007 - accuracy: 0.9684 - recall: 0.9568 - precision: 0.9774 - AUROC: 0.9992 - AUPRC: 0.9955
338/338 [==============================] - 2s 6ms/step - loss: 0.2313 - accuracy: 0.9267 - recall: 0.9128 - precision: 0.9420 - AUROC: 0.9947 - AUPRC: 0.9759
772/772 [==============================] - 2s 2ms/step
1it [06:01, 361.24s/it]
-- HOLDOUT 2
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_4 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_4 (MaxPooling (None, 40, 40, 128) 0
2D)
conv2d_5 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_5 (MaxPooling (None, 20, 20, 128) 0
2D)
conv2d_6 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_6 (MaxPooling (None, 10, 10, 256) 0
2D)
conv2d_7 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_7 (MaxPooling (None, 5, 5, 256) 0
2D)
flatten_1 (Flatten) (None, 6400) 0
dense_3 (Dense) (None, 128) 819328
dropout_2 (Dropout) (None, 128) 0
dense_4 (Dense) (None, 128) 16512
dropout_3 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
3086/3086 [==============================] - 32s 10ms/step - loss: 1.2249 - accuracy: 0.5262 - recall: 0.3379 - precision: 0.7239 - AUROC: 0.9049 - AUPRC: 0.5958 - val_loss: 0.6334 - val_accuracy: 0.7835 - val_recall: 0.7003 - val_precision: 0.8517 - val_AUROC: 0.9748 - val_AUPRC: 0.8658
Epoch 2/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.6799 - accuracy: 0.7666 - recall: 0.6848 - precision: 0.8333 - AUROC: 0.9695 - AUPRC: 0.8462 - val_loss: 0.4233 - val_accuracy: 0.8591 - val_recall: 0.8205 - val_precision: 0.8992 - val_AUROC: 0.9876 - val_AUPRC: 0.9317
Epoch 3/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.5093 - accuracy: 0.8307 - recall: 0.7814 - precision: 0.8756 - AUROC: 0.9815 - AUPRC: 0.9068 - val_loss: 0.3348 - val_accuracy: 0.8916 - val_recall: 0.8635 - val_precision: 0.9176 - val_AUROC: 0.9919 - val_AUPRC: 0.9546
Epoch 4/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.4266 - accuracy: 0.8610 - recall: 0.8244 - precision: 0.8955 - AUROC: 0.9860 - AUPRC: 0.9310 - val_loss: 0.3322 - val_accuracy: 0.8869 - val_recall: 0.8576 - val_precision: 0.9134 - val_AUROC: 0.9919 - val_AUPRC: 0.9551
Epoch 5/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.3735 - accuracy: 0.8776 - recall: 0.8491 - precision: 0.9074 - AUROC: 0.9891 - AUPRC: 0.9463 - val_loss: 0.3304 - val_accuracy: 0.8869 - val_recall: 0.8620 - val_precision: 0.9153 - val_AUROC: 0.9918 - val_AUPRC: 0.9566
Epoch 6/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3303 - accuracy: 0.8930 - recall: 0.8681 - precision: 0.9172 - AUROC: 0.9907 - AUPRC: 0.9549 - val_loss: 0.2683 - val_accuracy: 0.9073 - val_recall: 0.8889 - val_precision: 0.9266 - val_AUROC: 0.9939 - val_AUPRC: 0.9683
Epoch 7/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2991 - accuracy: 0.9032 - recall: 0.8831 - precision: 0.9255 - AUROC: 0.9920 - AUPRC: 0.9620 - val_loss: 0.2750 - val_accuracy: 0.9111 - val_recall: 0.8956 - val_precision: 0.9272 - val_AUROC: 0.9932 - val_AUPRC: 0.9665
Epoch 8/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2719 - accuracy: 0.9126 - recall: 0.8934 - precision: 0.9327 - AUROC: 0.9930 - AUPRC: 0.9676 - val_loss: 0.2615 - val_accuracy: 0.9175 - val_recall: 0.9019 - val_precision: 0.9336 - val_AUROC: 0.9932 - val_AUPRC: 0.9687
Epoch 9/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2618 - accuracy: 0.9163 - recall: 0.9001 - precision: 0.9348 - AUROC: 0.9935 - AUPRC: 0.9697 - val_loss: 0.2661 - val_accuracy: 0.9144 - val_recall: 0.8997 - val_precision: 0.9306 - val_AUROC: 0.9929 - val_AUPRC: 0.9676
Epoch 10/100
3086/3086 [==============================] - 32s 10ms/step - loss: 0.2378 - accuracy: 0.9241 - recall: 0.9101 - precision: 0.9406 - AUROC: 0.9944 - AUPRC: 0.9745 - val_loss: 0.2486 - val_accuracy: 0.9197 - val_recall: 0.9059 - val_precision: 0.9354 - val_AUROC: 0.9937 - val_AUPRC: 0.9722
Epoch 11/100
3086/3086 [==============================] - 34s 11ms/step - loss: 0.2211 - accuracy: 0.9309 - recall: 0.9170 - precision: 0.9449 - AUROC: 0.9948 - AUPRC: 0.9773 - val_loss: 0.2425 - val_accuracy: 0.9232 - val_recall: 0.9147 - val_precision: 0.9366 - val_AUROC: 0.9932 - val_AUPRC: 0.9725
Epoch 12/100
3086/3086 [==============================] - 33s 11ms/step - loss: 0.2205 - accuracy: 0.9322 - recall: 0.9189 - precision: 0.9476 - AUROC: 0.9948 - AUPRC: 0.9776 - val_loss: 0.2394 - val_accuracy: 0.9284 - val_recall: 0.9149 - val_precision: 0.9402 - val_AUROC: 0.9937 - val_AUPRC: 0.9730
Epoch 13/100
3086/3086 [==============================] - 34s 11ms/step - loss: 0.1942 - accuracy: 0.9396 - recall: 0.9273 - precision: 0.9526 - AUROC: 0.9956 - AUPRC: 0.9815 - val_loss: 0.2810 - val_accuracy: 0.9198 - val_recall: 0.9094 - val_precision: 0.9321 - val_AUROC: 0.9919 - val_AUPRC: 0.9678
Epoch 14/100
3086/3086 [==============================] - 33s 11ms/step - loss: 0.1886 - accuracy: 0.9414 - recall: 0.9299 - precision: 0.9540 - AUROC: 0.9957 - AUPRC: 0.9823 - val_loss: 0.2398 - val_accuracy: 0.9261 - val_recall: 0.9146 - val_precision: 0.9380 - val_AUROC: 0.9935 - val_AUPRC: 0.9736
1350/1350 [==============================] - 9s 6ms/step - loss: 0.0852 - accuracy: 0.9731 - recall: 0.9665 - precision: 0.9789 - AUROC: 0.9992 - AUPRC: 0.9960
338/338 [==============================] - 2s 6ms/step - loss: 0.2398 - accuracy: 0.9261 - recall: 0.9146 - precision: 0.9380 - AUROC: 0.9935 - AUPRC: 0.9736
772/772 [==============================] - 2s 3ms/step
2it [13:42, 411.15s/it]
In [24]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)
print(f"CNN Soccer Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN Soccer - Train history:")
plot_train_history(CNN_history)
print("-"*100)
CNN Soccer Metrics - 2-holdouts estimate: Accuracy : train - 0.9707747995853424 -- test - 0.9263888895511627 AUROC : train - 0.9991893470287323 -- test - 0.9941038489341736 AUPRC : train - 0.9957385659217834 -- test - 0.9747623205184937 -------------------------------------------------------------------------------- CNN Soccer - Train history:
----------------------------------------------------------------------------------------------------
In [64]:
# Soccer CNN Metrics
CNN_metrics
Out[64]:
[{'train_evaluation': {'loss': 0.10073351860046387,
'accuracy': 0.9684252142906189,
'recall': 0.9568045735359192,
'precision': 0.9774398803710938,
'AUROC': 0.9991887807846069,
'AUPRC': 0.9954752922058105},
'test_evaluation': {'loss': 0.23127177357673645,
'accuracy': 0.9266666769981384,
'recall': 0.9127777814865112,
'precision': 0.9419971108436584,
'AUROC': 0.9946945905685425,
'AUPRC': 0.9758864641189575}},
{'train_evaluation': {'loss': 0.08517780154943466,
'accuracy': 0.9731243848800659,
'recall': 0.9664807319641113,
'precision': 0.9788526296615601,
'AUROC': 0.9991899132728577,
'AUPRC': 0.9960018396377563},
'test_evaluation': {'loss': 0.23984651267528534,
'accuracy': 0.926111102104187,
'recall': 0.9146296381950378,
'precision': 0.9379925727844238,
'AUROC': 0.9935131072998047,
'AUPRC': 0.9736381769180298}}]
In [83]:
# Save results (save dictionary to basic_CNN_metrics.pkl file)
with open('results/basic_CNN_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [28]:
# Save Soccer CNN
soccer_CNN = CNN_models[0]
soccer_CNN.save_weights('modelWeights/soccerCNN_Model.h5')
Confusion Matrix - Soccer CNN¶
In [30]:
model_and_holdout_to_plot = 0
true_labels = tf.math.argmax(CNN_holdout_test_labels[model_and_holdout_to_plot], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_predictions[model_and_holdout_to_plot], axis=1)
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=9,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set(rc = {'figure.figsize':(10,10)})
plt.xlabel('Predicted Labels', fontsize=14)
plt.ylabel('True Labels', fontsize=14)
plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="13")
plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="13" )
plt.show()
Main training loop - Soccer CNN with Data Augmentation¶
- Generate augmented images equal to 20% of the original test split size for training.
- Normalize images' values in 0-1 range.
- Build and train the model over the training set and test it over the test set for the different holdouts.
In [23]:
epochs = 100
batch_size = 10
In [24]:
augmentation_rate = 0.2
In [25]:
print("---- Soccer CNN with Data Augmentation ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_all, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data
x_train, x_test = np.array([data_soccer_all[x] for x in train_indices]), np.array([data_soccer_all[x] for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
## Data Augmentation
# Select specified % of elements
n_data_to_augment = int(augmentation_rate * len(x_train))
selected_indices = np.random.choice(len(x_train), n_data_to_augment, replace=False)
selected_elements = x_train[selected_indices]
selected_labels = y_train.values[selected_indices]
# Apply data augmentation to the selected elements
augmented_elements = np.array([augment_image(x) for x in selected_elements])
# Insert the modified elements in the training data
x_train = np.insert(x_train, selected_indices, augmented_elements, axis=0)
y_train = pd.DataFrame(np.insert(y_train, selected_indices, selected_labels, axis=0))
# Normalization
x_train, x_test = np.array([x/255 for x in x_train]), np.array([x/255 for x in x_test])
# Build CNN
CNN = build_soccer_CNN(x_train[0].shape)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_model(
CNN,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size
)
predictions = get_predictions(CNN, x_test, batch_size=batch_size)
CNN_holdout_predictions.append(predictions)
CNN_holdout_test_labels.append(y_test)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(CNN)
---- Soccer CNN with Data Augmentation ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d (MaxPooling2D (None, 40, 40, 128) 0
)
conv2d_1 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_1 (MaxPooling (None, 20, 20, 128) 0
2D)
conv2d_2 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_2 (MaxPooling (None, 10, 10, 256) 0
2D)
conv2d_3 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_3 (MaxPooling (None, 5, 5, 256) 0
2D)
flatten (Flatten) (None, 6400) 0
dense (Dense) (None, 128) 819328
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 128) 16512
dropout_1 (Dropout) (None, 128) 0
dense_2 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
5184/5184 [==============================] - 52s 8ms/step - loss: 1.2681 - accuracy: 0.5088 - recall: 0.3092 - precision: 0.7106 - AUROC: 0.8968 - AUPRC: 0.5710 - val_loss: 0.6507 - val_accuracy: 0.7681 - val_recall: 0.6743 - val_precision: 0.8434 - val_AUROC: 0.9750 - val_AUPRC: 0.8560
Epoch 2/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.8268 - accuracy: 0.7079 - recall: 0.6022 - precision: 0.8033 - AUROC: 0.9564 - AUPRC: 0.7908 - val_loss: 0.5008 - val_accuracy: 0.8249 - val_recall: 0.7672 - val_precision: 0.8745 - val_AUROC: 0.9838 - val_AUPRC: 0.9075
Epoch 3/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.6854 - accuracy: 0.7634 - recall: 0.6863 - precision: 0.8397 - AUROC: 0.9690 - AUPRC: 0.8498 - val_loss: 0.3980 - val_accuracy: 0.8682 - val_recall: 0.8281 - val_precision: 0.9031 - val_AUROC: 0.9887 - val_AUPRC: 0.9390
Epoch 4/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.6047 - accuracy: 0.7959 - recall: 0.7324 - precision: 0.8601 - AUROC: 0.9752 - AUPRC: 0.8804 - val_loss: 0.3494 - val_accuracy: 0.8860 - val_recall: 0.8483 - val_precision: 0.9198 - val_AUROC: 0.9914 - val_AUPRC: 0.9509
Epoch 5/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5459 - accuracy: 0.8187 - recall: 0.7638 - precision: 0.8741 - AUROC: 0.9795 - AUPRC: 0.9005 - val_loss: 0.3935 - val_accuracy: 0.8651 - val_recall: 0.8245 - val_precision: 0.8999 - val_AUROC: 0.9889 - val_AUPRC: 0.9406
Epoch 6/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4967 - accuracy: 0.8339 - recall: 0.7865 - precision: 0.8871 - AUROC: 0.9823 - AUPRC: 0.9148 - val_loss: 0.2773 - val_accuracy: 0.9030 - val_recall: 0.8824 - val_precision: 0.9262 - val_AUROC: 0.9938 - val_AUPRC: 0.9675
Epoch 7/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4635 - accuracy: 0.8478 - recall: 0.8041 - precision: 0.8942 - AUROC: 0.9845 - AUPRC: 0.9252 - val_loss: 0.2787 - val_accuracy: 0.9073 - val_recall: 0.8816 - val_precision: 0.9293 - val_AUROC: 0.9939 - val_AUPRC: 0.9672
Epoch 8/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4358 - accuracy: 0.8578 - recall: 0.8179 - precision: 0.9011 - AUROC: 0.9859 - AUPRC: 0.9330 - val_loss: 0.2683 - val_accuracy: 0.9134 - val_recall: 0.8938 - val_precision: 0.9328 - val_AUROC: 0.9937 - val_AUPRC: 0.9696
Epoch 9/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4161 - accuracy: 0.8647 - recall: 0.8265 - precision: 0.9060 - AUROC: 0.9871 - AUPRC: 0.9386 - val_loss: 0.2930 - val_accuracy: 0.9053 - val_recall: 0.8738 - val_precision: 0.9348 - val_AUROC: 0.9931 - val_AUPRC: 0.9652
Epoch 10/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.3978 - accuracy: 0.8711 - recall: 0.8333 - precision: 0.9108 - AUROC: 0.9879 - AUPRC: 0.9424 - val_loss: 0.2729 - val_accuracy: 0.9145 - val_recall: 0.8967 - val_precision: 0.9341 - val_AUROC: 0.9930 - val_AUPRC: 0.9679
1620/1620 [==============================] - 10s 6ms/step - loss: 0.2302 - accuracy: 0.9232 - recall: 0.8887 - precision: 0.9547 - AUROC: 0.9964 - AUPRC: 0.9791
338/338 [==============================] - 2s 6ms/step - loss: 0.2729 - accuracy: 0.9145 - recall: 0.8966 - precision: 0.9341 - AUROC: 0.9930 - AUPRC: 0.9679
1080/1080 [==============================] - 2s 2ms/step
1it [08:12, 492.79s/it]
-- HOLDOUT 2
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_4 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_4 (MaxPooling (None, 40, 40, 128) 0
2D)
conv2d_5 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_5 (MaxPooling (None, 20, 20, 128) 0
2D)
conv2d_6 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_6 (MaxPooling (None, 10, 10, 256) 0
2D)
conv2d_7 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_7 (MaxPooling (None, 5, 5, 256) 0
2D)
flatten_1 (Flatten) (None, 6400) 0
dense_3 (Dense) (None, 128) 819328
dropout_2 (Dropout) (None, 128) 0
dense_4 (Dense) (None, 128) 16512
dropout_3 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
5184/5184 [==============================] - 43s 8ms/step - loss: 1.4094 - accuracy: 0.4294 - recall: 0.2111 - precision: 0.6709 - AUROC: 0.8688 - AUPRC: 0.4881 - val_loss: 0.8121 - val_accuracy: 0.6980 - val_recall: 0.5565 - val_precision: 0.8086 - val_AUROC: 0.9599 - val_AUPRC: 0.7904
Epoch 2/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.8983 - accuracy: 0.6779 - recall: 0.5478 - precision: 0.7837 - AUROC: 0.9489 - AUPRC: 0.7544 - val_loss: 0.5515 - val_accuracy: 0.8088 - val_recall: 0.7489 - val_precision: 0.8619 - val_AUROC: 0.9804 - val_AUPRC: 0.8936
Epoch 3/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.7300 - accuracy: 0.7464 - recall: 0.6587 - precision: 0.8249 - AUROC: 0.9653 - AUPRC: 0.8315 - val_loss: 0.4336 - val_accuracy: 0.8559 - val_recall: 0.8017 - val_precision: 0.9024 - val_AUROC: 0.9884 - val_AUPRC: 0.9334
Epoch 4/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.6428 - accuracy: 0.7793 - recall: 0.7102 - precision: 0.8483 - AUROC: 0.9724 - AUPRC: 0.8645 - val_loss: 0.4014 - val_accuracy: 0.8676 - val_recall: 0.8164 - val_precision: 0.9057 - val_AUROC: 0.9889 - val_AUPRC: 0.9386
Epoch 5/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5860 - accuracy: 0.8023 - recall: 0.7401 - precision: 0.8632 - AUROC: 0.9767 - AUPRC: 0.8860 - val_loss: 0.3828 - val_accuracy: 0.8754 - val_recall: 0.8373 - val_precision: 0.9098 - val_AUROC: 0.9887 - val_AUPRC: 0.9421
Epoch 6/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5504 - accuracy: 0.8147 - recall: 0.7576 - precision: 0.8732 - AUROC: 0.9792 - AUPRC: 0.8979 - val_loss: 0.3977 - val_accuracy: 0.8641 - val_recall: 0.8244 - val_precision: 0.8995 - val_AUROC: 0.9885 - val_AUPRC: 0.9392
Epoch 7/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5244 - accuracy: 0.8244 - recall: 0.7702 - precision: 0.8796 - AUROC: 0.9809 - AUPRC: 0.9068 - val_loss: 0.3704 - val_accuracy: 0.8731 - val_recall: 0.8331 - val_precision: 0.9118 - val_AUROC: 0.9897 - val_AUPRC: 0.9455
Epoch 8/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4926 - accuracy: 0.8361 - recall: 0.7870 - precision: 0.8882 - AUROC: 0.9828 - AUPRC: 0.9165 - val_loss: 0.3330 - val_accuracy: 0.8944 - val_recall: 0.8536 - val_precision: 0.9275 - val_AUROC: 0.9914 - val_AUPRC: 0.9545
Epoch 9/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4753 - accuracy: 0.8437 - recall: 0.7958 - precision: 0.8931 - AUROC: 0.9837 - AUPRC: 0.9213 - val_loss: 0.2953 - val_accuracy: 0.9052 - val_recall: 0.8709 - val_precision: 0.9316 - val_AUROC: 0.9933 - val_AUPRC: 0.9639
Epoch 10/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.4521 - accuracy: 0.8500 - recall: 0.8058 - precision: 0.8969 - AUROC: 0.9850 - AUPRC: 0.9278 - val_loss: 0.2753 - val_accuracy: 0.9100 - val_recall: 0.8869 - val_precision: 0.9328 - val_AUROC: 0.9935 - val_AUPRC: 0.9676
Epoch 11/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4338 - accuracy: 0.8580 - recall: 0.8160 - precision: 0.9029 - AUROC: 0.9860 - AUPRC: 0.9332 - val_loss: 0.2979 - val_accuracy: 0.9047 - val_recall: 0.8749 - val_precision: 0.9307 - val_AUROC: 0.9928 - val_AUPRC: 0.9634
Epoch 12/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4214 - accuracy: 0.8626 - recall: 0.8217 - precision: 0.9068 - AUROC: 0.9867 - AUPRC: 0.9368 - val_loss: 0.3141 - val_accuracy: 0.8957 - val_recall: 0.8652 - val_precision: 0.9244 - val_AUROC: 0.9920 - val_AUPRC: 0.9587
1620/1620 [==============================] - 10s 6ms/step - loss: 0.2824 - accuracy: 0.9041 - recall: 0.8597 - precision: 0.9442 - AUROC: 0.9950 - AUPRC: 0.9700
338/338 [==============================] - 2s 6ms/step - loss: 0.3141 - accuracy: 0.8957 - recall: 0.8652 - precision: 0.9244 - AUROC: 0.9920 - AUPRC: 0.9587
1080/1080 [==============================] - 2s 2ms/step
2it [17:42, 531.15s/it]
In [26]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)
print(f"CNN Soccer Data Augmentation - Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN Soccer Data Augmentation - Train history:")
plot_train_history(CNN_history)
print("-"*100)
CNN Soccer Data Augmentation - Metrics - 2-holdouts estimate: Accuracy : train - 0.9136347770690918 -- test - 0.9051388800144196 AUROC : train - 0.9957026243209839 -- test - 0.9924991726875305 AUPRC : train - 0.9745860695838928 -- test - 0.9633217751979828 -------------------------------------------------------------------------------- CNN Soccer Data Augmentation - Train history:
----------------------------------------------------------------------------------------------------
In [27]:
# Soccer CNN Data augmentation Metrics
CNN_metrics
Out[27]:
[{'train_evaluation': {'loss': 0.23024460673332214,
'accuracy': 0.9232030510902405,
'recall': 0.8886530995368958,
'precision': 0.9546961784362793,
'AUROC': 0.9964159727096558,
'AUPRC': 0.9791415929794312},
'test_evaluation': {'loss': 0.272890567779541,
'accuracy': 0.9145370125770569,
'recall': 0.896574079990387,
'precision': 0.9341115355491638,
'AUROC': 0.9929803609848022,
'AUPRC': 0.9679373502731323}},
{'train_evaluation': {'loss': 0.2823828458786011,
'accuracy': 0.9040665030479431,
'recall': 0.85965895652771,
'precision': 0.9441913962364197,
'AUROC': 0.994989275932312,
'AUPRC': 0.9700305461883545},
'test_evaluation': {'loss': 0.31406882405281067,
'accuracy': 0.8957407474517822,
'recall': 0.8651852011680603,
'precision': 0.9244163036346436,
'AUROC': 0.9920179843902588,
'AUPRC': 0.9587062001228333}}]
In [85]:
# Save results (save dictionary to data_augmentation_basic_CNN_metrics.pkl file)
with open('results/data_augmentation_basic_CNN_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [28]:
# Save Soccer CNN with Data Augmentation
soccer_data_augmentation_CNN = CNN_models[0]
soccer_data_augmentation_CNN.save_weights('modelWeights/soccerModelDataAugmentation.h5')
Transfer Learning¶
Import ResNet50 and VGG16 architectures pretrained (imagenet weights) and add an average pooling layer as well as a dense layer with dropout.
- ResNet50's final 3 layers are trained as well as the added ones.
- VGG16 is instead kept fully frozen and only the added layers are trained.
The models require three channel images, thus they are tested on gray scale images reshaped to have three channels as well as RGB images.
In [23]:
# 3 channels gray input
threeChannels_input = [cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in data_soccer_all]
image_shape = threeChannels_input[0].shape
image_shape
Out[23]:
(80, 80, 3)
In [24]:
# RGB input
#data_soccer is a dict containing ['events'...] so put together classes to evaluate
data_soccer_rgb_all = []
for event in data_soccer_rgb:
for element in data_soccer_rgb[event]:
data_soccer_rgb_all.append(element)
print(f"RGB Data shape: {np.array(data_soccer_rgb_all).shape}")
rgb_input = data_soccer_rgb_all
image_shape = rgb_input[0].shape
image_shape
RGB Data shape: (53999, 80, 80, 3)
Out[24]:
(80, 80, 3)
In [25]:
# Create the base model from the pre-trained model MobileNet V2
resNet_model = tf.keras.applications.resnet.ResNet50(input_shape=image_shape,
include_top=False,
weights='imagenet')
In [26]:
# Create the base model from the pre-trained model VGG16
vgg16_model = tf.keras.applications.vgg16.VGG16(input_shape=image_shape,
include_top=False,
weights='imagenet')
In [27]:
# Freeze pretrained layers
resNet_model.trainable = False
vgg16_model.trainable = False
In [28]:
resNet_model.trainable = True
print("Number of layers in the base model: ", len(resNet_model.layers))
# Fine-tune from this layer onwards
fine_tune_at = 172
# Freeze all the layers before the `fine_tune_at` layer
for layer in resNet_model.layers[:fine_tune_at]:
layer.trainable = False
print(f"Fine tuning from layer: {fine_tune_at}")
Number of layers in the base model: 175 Fine tuning from layer: 172
In [29]:
# Additional downstream layers on top of main model
avg_pooling = layers.GlobalAveragePooling2D()
dense1 = layers.Dense(units = 256, activation='relu')
dropout1 = layers.Dropout(0.5)
prediction_layer = layers.Dense(9, activation='softmax')
In [30]:
# Build final model ResNet
inputs = tf.keras.Input(shape=(soccer_images_dim, soccer_images_dim, 3))
x = resNet_model(inputs)
x = avg_pooling(x)
x = dense1(x)
x = dropout1(x)
outputs = prediction_layer(x)
resNet_soccer_model = tf.keras.Model(inputs, outputs)
resNet_soccer_model.summary()
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_3 (InputLayer) [(None, 80, 80, 3)] 0
resnet50 (Functional) (None, 3, 3, 2048) 23587712
global_average_pooling2d (G (None, 2048) 0
lobalAveragePooling2D)
dense (Dense) (None, 256) 524544
dropout (Dropout) (None, 256) 0
dense_1 (Dense) (None, 9) 2313
=================================================================
Total params: 24,114,569
Trainable params: 530,953
Non-trainable params: 23,583,616
_________________________________________________________________
In [31]:
resNet_soccer_model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=get_minimal_multiclass_metrics()
)
Main training loop - ResNet CNN (Gray and RGB images)¶
- Normalize images' values in 0-1 range.
- Train the unfrozen layers and additional ones of the ResNet model over the training set and test it over the test set for the different holdouts.
- For memory and speed requirements these models are run over a single holdout and trained using checkpoints.
In [33]:
number_of_splits = 1
holdouts_generator = StratifiedShuffleSplit(
n_splits = number_of_splits,
test_size = 0.2,
random_state = 42
)
In [34]:
epochs = 100
batch_size = 24
In [37]:
checkpoint_path = "training_ResNet_gray/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
save_best_only=True,
verbose=1)
interrupt = False
enable_checkpoints = True
In [38]:
# Flag to load weights if training got interrupted
interrupt = True
In [39]:
print("---- ResNet Soccer CNN Gray ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(threeChannels_input, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([threeChannels_input[x]/255 for x in train_indices]), np.array([threeChannels_input[x]/255 for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
# Get model
model = resNet_soccer_model
# Load weights if training got interrupted
if interrupt:
model.load_weights(checkpoint_path)
CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
model,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size,
cp_callback
)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(model)
---- ResNet Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1 Epoch 1/100 5400/5400 [==============================] - ETA: 0s - loss: 0.7055 - accuracy: 0.7354 - recall: 0.6543 - precision: 0.8096 - AUROC: 0.9677 - AUPRC: 0.8299 Epoch 1: val_loss improved from inf to 0.64415, saving model to training_ResNet_gray\cp.ckpt 5400/5400 [==============================] - 85s 14ms/step - loss: 0.7055 - accuracy: 0.7354 - recall: 0.6543 - precision: 0.8096 - AUROC: 0.9677 - AUPRC: 0.8299 - val_loss: 0.6442 - val_accuracy: 0.7766 - val_recall: 0.7235 - val_precision: 0.8237 - val_AUROC: 0.9721 - val_AUPRC: 0.8570 Epoch 2/100 5396/5400 [============================>.] - ETA: 0s - loss: 0.7060 - accuracy: 0.7368 - recall: 0.6550 - precision: 0.8119 - AUROC: 0.9676 - AUPRC: 0.8302 Epoch 2: val_loss improved from 0.64415 to 0.63572, saving model to training_ResNet_gray\cp.ckpt 5400/5400 [==============================] - 71s 13ms/step - loss: 0.7061 - accuracy: 0.7367 - recall: 0.6550 - precision: 0.8119 - AUROC: 0.9676 - AUPRC: 0.8302 - val_loss: 0.6357 - val_accuracy: 0.7869 - val_recall: 0.7277 - val_precision: 0.8328 - val_AUROC: 0.9731 - val_AUPRC: 0.8615 Epoch 3/100 5397/5400 [============================>.] - ETA: 0s - loss: 0.7067 - accuracy: 0.7374 - recall: 0.6565 - precision: 0.8128 - AUROC: 0.9677 - AUPRC: 0.8303 Epoch 3: val_loss improved from 0.63572 to 0.62216, saving model to training_ResNet_gray\cp.ckpt 5400/5400 [==============================] - 72s 13ms/step - loss: 0.7067 - accuracy: 0.7374 - recall: 0.6565 - precision: 0.8128 - AUROC: 0.9677 - AUPRC: 0.8303 - val_loss: 0.6222 - val_accuracy: 0.7818 - val_recall: 0.7328 - val_precision: 0.8293 - val_AUROC: 0.9740 - val_AUPRC: 0.8662 Epoch 4/100 5397/5400 [============================>.] - ETA: 0s - loss: 0.7070 - accuracy: 0.7377 - recall: 0.6572 - precision: 0.8122 - AUROC: 0.9674 - AUPRC: 0.8299 Epoch 4: val_loss did not improve from 0.62216 5400/5400 [==============================] - 67s 12ms/step - loss: 0.7071 - accuracy: 0.7376 - recall: 0.6571 - precision: 0.8122 - AUROC: 0.9674 - AUPRC: 0.8299 - val_loss: 0.6446 - val_accuracy: 0.7794 - val_recall: 0.7085 - val_precision: 0.8388 - val_AUROC: 0.9728 - val_AUPRC: 0.8582 Epoch 5/100 5397/5400 [============================>.] - ETA: 0s - loss: 0.6991 - accuracy: 0.7382 - recall: 0.6585 - precision: 0.8125 - AUROC: 0.9683 - AUPRC: 0.8321 Epoch 5: val_loss improved from 0.62216 to 0.60718, saving model to training_ResNet_gray\cp.ckpt 5400/5400 [==============================] - 66s 12ms/step - loss: 0.6991 - accuracy: 0.7382 - recall: 0.6586 - precision: 0.8125 - AUROC: 0.9683 - AUPRC: 0.8321 - val_loss: 0.6072 - val_accuracy: 0.7924 - val_recall: 0.7368 - val_precision: 0.8367 - val_AUROC: 0.9752 - val_AUPRC: 0.8709 Epoch 6/100 5398/5400 [============================>.] - ETA: 0s - loss: 0.7042 - accuracy: 0.7347 - recall: 0.6549 - precision: 0.8095 - AUROC: 0.9678 - AUPRC: 0.8300 Epoch 6: val_loss did not improve from 0.60718 5400/5400 [==============================] - 68s 13ms/step - loss: 0.7044 - accuracy: 0.7347 - recall: 0.6549 - precision: 0.8095 - AUROC: 0.9678 - AUPRC: 0.8299 - val_loss: 0.6132 - val_accuracy: 0.7903 - val_recall: 0.7395 - val_precision: 0.8333 - val_AUROC: 0.9750 - val_AUPRC: 0.8715 Epoch 7/100 5400/5400 [==============================] - ETA: 0s - loss: 0.6988 - accuracy: 0.7410 - recall: 0.6609 - precision: 0.8141 - AUROC: 0.9683 - AUPRC: 0.8325 Epoch 7: val_loss did not improve from 0.60718 5400/5400 [==============================] - 70s 13ms/step - loss: 0.6988 - accuracy: 0.7410 - recall: 0.6609 - precision: 0.8141 - AUROC: 0.9683 - AUPRC: 0.8325 - val_loss: 0.6257 - val_accuracy: 0.7849 - val_recall: 0.7348 - val_precision: 0.8291 - val_AUROC: 0.9736 - val_AUPRC: 0.8631 338/338 [==============================] - 5s 13ms/step - loss: 0.6257 - accuracy: 0.7856 - recall: 0.7341 - precision: 0.8293 - AUROC: 0.9736 - AUPRC: 0.8633
1it [09:04, 544.35s/it]
In [40]:
print(f"ResNet Soccer CNN Gray Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate: Accuracy : test - 0.7855555415153503 AUROC : test - 0.9735950231552124 AUPRC : test - 0.8632519245147705 --------------------------------------------------------------------------------
In [42]:
# CNN ResNet - Gray images - Metrics
CNN_metrics
Out[42]:
[{'test_evaluation': {'loss': 0.6256866455078125,
'accuracy': 0.7855555415153503,
'recall': 0.734074056148529,
'precision': 0.8292887210845947,
'AUROC': 0.9735950231552124,
'AUPRC': 0.8632519245147705}}]
In [71]:
# Save results (save dictionary to resNet_gray_metrics.pkl file)
with open('results/resNet_gray_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [44]:
# Save Gray ResNet Soccer CNN
resNet_soccer_gray = CNN_models[0]
resNet_soccer_gray.save_weights('modelWeights/resNet_soccer_gray_model.h5')
In [35]:
checkpoint_path = "training_ResNet_rgb/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
save_best_only=True,
verbose=1)
interrupt = False
In [36]:
# Flag to load weights if training got interrupted
interrupt = True
In [37]:
print("---- ResNet Soccer CNN RGB ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(rgb_input, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([rgb_input[x]/255 for x in train_indices]), np.array([rgb_input[x]/255 for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
# Get model
model = resNet_soccer_model
# Load weights if training got interrupted
if interrupt:
model.load_weights(checkpoint_path)
CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
model,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size,
cp_callback
)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(model)
---- ResNet Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1 Epoch 1/100 1800/1800 [==============================] - ETA: 0s - loss: 0.8205 - accuracy: 0.6941 - recall: 0.5868 - precision: 0.7885 - AUROC: 0.9570 - AUPRC: 0.7813 Epoch 1: val_loss improved from inf to 0.69696, saving model to training_ResNet_rgb\cp.ckpt 1800/1800 [==============================] - 44s 19ms/step - loss: 0.8205 - accuracy: 0.6941 - recall: 0.5868 - precision: 0.7885 - AUROC: 0.9570 - AUPRC: 0.7813 - val_loss: 0.6970 - val_accuracy: 0.7419 - val_recall: 0.6651 - val_precision: 0.8115 - val_AUROC: 0.9685 - val_AUPRC: 0.8311 Epoch 2/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.8157 - accuracy: 0.6990 - recall: 0.5889 - precision: 0.7887 - AUROC: 0.9575 - AUPRC: 0.7829 Epoch 2: val_loss improved from 0.69696 to 0.67831, saving model to training_ResNet_rgb\cp.ckpt 1800/1800 [==============================] - 30s 17ms/step - loss: 0.8156 - accuracy: 0.6991 - recall: 0.5890 - precision: 0.7888 - AUROC: 0.9575 - AUPRC: 0.7830 - val_loss: 0.6783 - val_accuracy: 0.7570 - val_recall: 0.6832 - val_precision: 0.8214 - val_AUROC: 0.9699 - val_AUPRC: 0.8398 Epoch 3/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.8062 - accuracy: 0.7009 - recall: 0.5966 - precision: 0.7927 - AUROC: 0.9584 - AUPRC: 0.7876 Epoch 3: val_loss improved from 0.67831 to 0.65387, saving model to training_ResNet_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.8058 - accuracy: 0.7011 - recall: 0.5968 - precision: 0.7928 - AUROC: 0.9584 - AUPRC: 0.7878 - val_loss: 0.6539 - val_accuracy: 0.7609 - val_recall: 0.6978 - val_precision: 0.8247 - val_AUROC: 0.9719 - val_AUPRC: 0.8502 Epoch 4/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.8027 - accuracy: 0.7031 - recall: 0.5989 - precision: 0.7911 - AUROC: 0.9587 - AUPRC: 0.7886 Epoch 4: val_loss did not improve from 0.65387 1800/1800 [==============================] - 28s 16ms/step - loss: 0.8026 - accuracy: 0.7031 - recall: 0.5988 - precision: 0.7910 - AUROC: 0.9587 - AUPRC: 0.7885 - val_loss: 0.6727 - val_accuracy: 0.7613 - val_recall: 0.6639 - val_precision: 0.8455 - val_AUROC: 0.9717 - val_AUPRC: 0.8477 Epoch 5/100 1800/1800 [==============================] - ETA: 0s - loss: 0.7865 - accuracy: 0.7112 - recall: 0.6095 - precision: 0.7949 - AUROC: 0.9604 - AUPRC: 0.7956 Epoch 5: val_loss did not improve from 0.65387 1800/1800 [==============================] - 28s 16ms/step - loss: 0.7865 - accuracy: 0.7112 - recall: 0.6095 - precision: 0.7949 - AUROC: 0.9604 - AUPRC: 0.7956 - val_loss: 0.6634 - val_accuracy: 0.7576 - val_recall: 0.6902 - val_precision: 0.8189 - val_AUROC: 0.9711 - val_AUPRC: 0.8446 338/338 [==============================] - 5s 13ms/step - loss: 0.6637 - accuracy: 0.7575 - recall: 0.6895 - precision: 0.8184 - AUROC: 0.9711 - AUPRC: 0.8445
1it [03:22, 202.54s/it]
In [38]:
print(f"ResNet Soccer CNN RGB Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate: Accuracy : test - 0.7574999928474426 AUROC : test - 0.9711117744445801 AUPRC : test - 0.8445291519165039 --------------------------------------------------------------------------------
In [39]:
# CNN ResNet - RGB images - Metrics
CNN_metrics
Out[39]:
[{'test_evaluation': {'loss': 0.663693368434906,
'accuracy': 0.7574999928474426,
'recall': 0.6895370483398438,
'precision': 0.8184415698051453,
'AUROC': 0.9711117744445801,
'AUPRC': 0.8445291519165039}}]
In [73]:
# Save results (save dictionary to resNet_rgb_metrics.pkl file)
with open('results/resNet_rgb_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [40]:
# Save RGB ResNet Soccer CNN
resNet_soccer_rgb = CNN_models[0]
resNet_soccer_rgb.save_weights('modelWeights/resNet_soccer_rgb_model.h5')
Main training loop - VGG16 CNN (Gray and RGB images)¶
- Normalize images' values in 0-1 range.
- Train the additional layers of the VGG16 model over the training set and test it over the test set for the different holdouts.
In [32]:
vgg16_model.trainable = True
print("Number of layers in the base model: ", len(vgg16_model.layers))
# Fine-tune from this layer onwards
fine_tune_at = 19
# Freeze all the layers before the `fine_tune_at` layer
for layer in vgg16_model.layers[:fine_tune_at]:
layer.trainable = False
Number of layers in the base model: 19
In [33]:
vgg16_model.summary()
Model: "vgg16"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_2 (InputLayer) [(None, 80, 80, 3)] 0
block1_conv1 (Conv2D) (None, 80, 80, 64) 1792
block1_conv2 (Conv2D) (None, 80, 80, 64) 36928
block1_pool (MaxPooling2D) (None, 40, 40, 64) 0
block2_conv1 (Conv2D) (None, 40, 40, 128) 73856
block2_conv2 (Conv2D) (None, 40, 40, 128) 147584
block2_pool (MaxPooling2D) (None, 20, 20, 128) 0
block3_conv1 (Conv2D) (None, 20, 20, 256) 295168
block3_conv2 (Conv2D) (None, 20, 20, 256) 590080
block3_conv3 (Conv2D) (None, 20, 20, 256) 590080
block3_pool (MaxPooling2D) (None, 10, 10, 256) 0
block4_conv1 (Conv2D) (None, 10, 10, 512) 1180160
block4_conv2 (Conv2D) (None, 10, 10, 512) 2359808
block4_conv3 (Conv2D) (None, 10, 10, 512) 2359808
block4_pool (MaxPooling2D) (None, 5, 5, 512) 0
block5_conv1 (Conv2D) (None, 5, 5, 512) 2359808
block5_conv2 (Conv2D) (None, 5, 5, 512) 2359808
block5_conv3 (Conv2D) (None, 5, 5, 512) 2359808
block5_pool (MaxPooling2D) (None, 2, 2, 512) 0
=================================================================
Total params: 14,714,688
Trainable params: 0
Non-trainable params: 14,714,688
_________________________________________________________________
In [34]:
# Additional downstream layers on top of main model
avg_pooling = layers.GlobalAveragePooling2D()
flattening = layers.Flatten()
dense1 = layers.Dense(units = 256, activation='relu')
dropout1 = layers.Dropout(0.5)
dense2 = layers.Dense(units = 128, activation='relu')
dropout2 = layers.Dropout(0.5)
prediction_layer = layers.Dense(9, activation='softmax')
In [35]:
# Build final model VGG16
inputs = tf.keras.Input(shape=(soccer_images_dim, soccer_images_dim, 3))
x = vgg16_model(inputs, training=False)
x = avg_pooling(x)
x = dense1(x)
x = dropout1(x)
outputs = prediction_layer(x)
vgg16_soccer_model = tf.keras.Model(inputs, outputs)
vgg16_soccer_model.summary()
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_4 (InputLayer) [(None, 80, 80, 3)] 0
vgg16 (Functional) (None, 2, 2, 512) 14714688
global_average_pooling2d_1 (None, 512) 0
(GlobalAveragePooling2D)
dense_2 (Dense) (None, 256) 131328
dropout_1 (Dropout) (None, 256) 0
dense_4 (Dense) (None, 9) 2313
=================================================================
Total params: 14,848,329
Trainable params: 133,641
Non-trainable params: 14,714,688
_________________________________________________________________
In [36]:
vgg16_soccer_model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=get_minimal_multiclass_metrics()
)
Main training loop - VGG16 CNN (Gray and RGB images)¶
- Normalize images' values in 0-1 range.
- Train the unfrozen layers and additional ones of the VGG16 model over the training set and test it over the test set for the different holdouts.
In [41]:
checkpoint_path = "training_vgg_gray/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
save_best_only=True,
verbose=1)
interrupt = False
In [ ]:
# Flag to load weights if training got interrupted
interrupt = True
In [42]:
print("---- VGG16 Soccer CNN Gray ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(threeChannels_input, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([threeChannels_input[x]/255 for x in train_indices]), np.array([threeChannels_input[x]/255 for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
# Get model
model = vgg16_soccer_model
# Load weights if training got interrupted
if interrupt:
model.load_weights(checkpoint_path)
CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
model,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size,
cp_callback
)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(model)
---- VGG16 Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1 Epoch 1/100 1800/1800 [==============================] - ETA: 0s - loss: 0.9374 - accuracy: 0.6603 - recall: 0.5088 - precision: 0.7808 - AUROC: 0.9453 - AUPRC: 0.7365 Epoch 1: val_loss improved from inf to 0.67921, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 41s 18ms/step - loss: 0.9374 - accuracy: 0.6603 - recall: 0.5088 - precision: 0.7808 - AUROC: 0.9453 - AUPRC: 0.7365 - val_loss: 0.6792 - val_accuracy: 0.7593 - val_recall: 0.6690 - val_precision: 0.8331 - val_AUROC: 0.9710 - val_AUPRC: 0.8434 Epoch 2/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.7039 - accuracy: 0.7480 - recall: 0.6654 - precision: 0.8144 - AUROC: 0.9681 - AUPRC: 0.8307 Epoch 2: val_loss improved from 0.67921 to 0.62108, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.7041 - accuracy: 0.7479 - recall: 0.6653 - precision: 0.8143 - AUROC: 0.9681 - AUPRC: 0.8306 - val_loss: 0.6211 - val_accuracy: 0.7749 - val_recall: 0.6942 - val_precision: 0.8382 - val_AUROC: 0.9754 - val_AUPRC: 0.8648 Epoch 3/100 1800/1800 [==============================] - ETA: 0s - loss: 0.6365 - accuracy: 0.7702 - recall: 0.7033 - precision: 0.8265 - AUROC: 0.9737 - AUPRC: 0.8559 Epoch 3: val_loss improved from 0.62108 to 0.56647, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.6365 - accuracy: 0.7702 - recall: 0.7033 - precision: 0.8265 - AUROC: 0.9737 - AUPRC: 0.8559 - val_loss: 0.5665 - val_accuracy: 0.7948 - val_recall: 0.7282 - val_precision: 0.8531 - val_AUROC: 0.9789 - val_AUPRC: 0.8826 Epoch 4/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.5948 - accuracy: 0.7859 - recall: 0.7279 - precision: 0.8353 - AUROC: 0.9763 - AUPRC: 0.8707 Epoch 4: val_loss improved from 0.56647 to 0.54031, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5948 - accuracy: 0.7859 - recall: 0.7280 - precision: 0.8352 - AUROC: 0.9763 - AUPRC: 0.8707 - val_loss: 0.5403 - val_accuracy: 0.8046 - val_recall: 0.7635 - val_precision: 0.8444 - val_AUROC: 0.9802 - val_AUPRC: 0.8899 Epoch 5/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.5660 - accuracy: 0.7943 - recall: 0.7433 - precision: 0.8426 - AUROC: 0.9785 - AUPRC: 0.8806 Epoch 5: val_loss improved from 0.54031 to 0.49853, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5660 - accuracy: 0.7943 - recall: 0.7433 - precision: 0.8426 - AUROC: 0.9785 - AUPRC: 0.8806 - val_loss: 0.4985 - val_accuracy: 0.8217 - val_recall: 0.7799 - val_precision: 0.8611 - val_AUROC: 0.9830 - val_AUPRC: 0.9055 Epoch 6/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.5360 - accuracy: 0.8064 - recall: 0.7572 - precision: 0.8487 - AUROC: 0.9805 - AUPRC: 0.8913 Epoch 6: val_loss improved from 0.49853 to 0.48654, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5359 - accuracy: 0.8064 - recall: 0.7573 - precision: 0.8487 - AUROC: 0.9805 - AUPRC: 0.8913 - val_loss: 0.4865 - val_accuracy: 0.8242 - val_recall: 0.7819 - val_precision: 0.8603 - val_AUROC: 0.9836 - val_AUPRC: 0.9082 Epoch 7/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.5176 - accuracy: 0.8114 - recall: 0.7657 - precision: 0.8527 - AUROC: 0.9817 - AUPRC: 0.8979 Epoch 7: val_loss improved from 0.48654 to 0.46898, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5177 - accuracy: 0.8114 - recall: 0.7657 - precision: 0.8526 - AUROC: 0.9817 - AUPRC: 0.8979 - val_loss: 0.4690 - val_accuracy: 0.8329 - val_recall: 0.7958 - val_precision: 0.8635 - val_AUROC: 0.9846 - val_AUPRC: 0.9130 Epoch 8/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.5019 - accuracy: 0.8185 - recall: 0.7752 - precision: 0.8575 - AUROC: 0.9828 - AUPRC: 0.9029 Epoch 8: val_loss improved from 0.46898 to 0.46023, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5019 - accuracy: 0.8185 - recall: 0.7752 - precision: 0.8575 - AUROC: 0.9828 - AUPRC: 0.9029 - val_loss: 0.4602 - val_accuracy: 0.8369 - val_recall: 0.7983 - val_precision: 0.8700 - val_AUROC: 0.9848 - val_AUPRC: 0.9165 Epoch 9/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.4862 - accuracy: 0.8229 - recall: 0.7818 - precision: 0.8607 - AUROC: 0.9838 - AUPRC: 0.9081 Epoch 9: val_loss improved from 0.46023 to 0.45084, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4861 - accuracy: 0.8230 - recall: 0.7819 - precision: 0.8607 - AUROC: 0.9838 - AUPRC: 0.9081 - val_loss: 0.4508 - val_accuracy: 0.8375 - val_recall: 0.8064 - val_precision: 0.8699 - val_AUROC: 0.9856 - val_AUPRC: 0.9198 Epoch 10/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4745 - accuracy: 0.8261 - recall: 0.7883 - precision: 0.8622 - AUROC: 0.9844 - AUPRC: 0.9119 Epoch 10: val_loss improved from 0.45084 to 0.44901, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4745 - accuracy: 0.8261 - recall: 0.7883 - precision: 0.8622 - AUROC: 0.9844 - AUPRC: 0.9119 - val_loss: 0.4490 - val_accuracy: 0.8398 - val_recall: 0.8027 - val_precision: 0.8737 - val_AUROC: 0.9855 - val_AUPRC: 0.9199 Epoch 11/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4621 - accuracy: 0.8312 - recall: 0.7958 - precision: 0.8659 - AUROC: 0.9851 - AUPRC: 0.9153 Epoch 11: val_loss improved from 0.44901 to 0.44684, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 28s 16ms/step - loss: 0.4621 - accuracy: 0.8312 - recall: 0.7958 - precision: 0.8659 - AUROC: 0.9851 - AUPRC: 0.9153 - val_loss: 0.4468 - val_accuracy: 0.8400 - val_recall: 0.8036 - val_precision: 0.8733 - val_AUROC: 0.9857 - val_AUPRC: 0.9211 Epoch 12/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4504 - accuracy: 0.8352 - recall: 0.8000 - precision: 0.8670 - AUROC: 0.9858 - AUPRC: 0.9193 Epoch 12: val_loss improved from 0.44684 to 0.44189, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4504 - accuracy: 0.8352 - recall: 0.8000 - precision: 0.8670 - AUROC: 0.9858 - AUPRC: 0.9193 - val_loss: 0.4419 - val_accuracy: 0.8422 - val_recall: 0.8119 - val_precision: 0.8738 - val_AUROC: 0.9860 - val_AUPRC: 0.9213 Epoch 13/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.4418 - accuracy: 0.8390 - recall: 0.8054 - precision: 0.8700 - AUROC: 0.9863 - AUPRC: 0.9217 Epoch 13: val_loss improved from 0.44189 to 0.42453, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4419 - accuracy: 0.8390 - recall: 0.8054 - precision: 0.8700 - AUROC: 0.9863 - AUPRC: 0.9216 - val_loss: 0.4245 - val_accuracy: 0.8494 - val_recall: 0.8214 - val_precision: 0.8761 - val_AUROC: 0.9865 - val_AUPRC: 0.9271 Epoch 14/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.4351 - accuracy: 0.8402 - recall: 0.8080 - precision: 0.8720 - AUROC: 0.9867 - AUPRC: 0.9236 Epoch 14: val_loss improved from 0.42453 to 0.42399, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4351 - accuracy: 0.8402 - recall: 0.8080 - precision: 0.8720 - AUROC: 0.9867 - AUPRC: 0.9236 - val_loss: 0.4240 - val_accuracy: 0.8468 - val_recall: 0.8141 - val_precision: 0.8763 - val_AUROC: 0.9873 - val_AUPRC: 0.9278 Epoch 15/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.4281 - accuracy: 0.8439 - recall: 0.8112 - precision: 0.8733 - AUROC: 0.9870 - AUPRC: 0.9257 Epoch 15: val_loss improved from 0.42399 to 0.41646, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4282 - accuracy: 0.8438 - recall: 0.8112 - precision: 0.8732 - AUROC: 0.9870 - AUPRC: 0.9257 - val_loss: 0.4165 - val_accuracy: 0.8502 - val_recall: 0.8215 - val_precision: 0.8822 - val_AUROC: 0.9874 - val_AUPRC: 0.9299 Epoch 16/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4174 - accuracy: 0.8464 - recall: 0.8161 - precision: 0.8754 - AUROC: 0.9877 - AUPRC: 0.9289 Epoch 16: val_loss did not improve from 0.41646 1800/1800 [==============================] - 28s 15ms/step - loss: 0.4174 - accuracy: 0.8464 - recall: 0.8161 - precision: 0.8754 - AUROC: 0.9877 - AUPRC: 0.9289 - val_loss: 0.4189 - val_accuracy: 0.8547 - val_recall: 0.8271 - val_precision: 0.8768 - val_AUROC: 0.9870 - val_AUPRC: 0.9290 Epoch 17/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4108 - accuracy: 0.8499 - recall: 0.8216 - precision: 0.8779 - AUROC: 0.9880 - AUPRC: 0.9309 Epoch 17: val_loss improved from 0.41646 to 0.40520, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 28s 16ms/step - loss: 0.4108 - accuracy: 0.8499 - recall: 0.8216 - precision: 0.8779 - AUROC: 0.9880 - AUPRC: 0.9309 - val_loss: 0.4052 - val_accuracy: 0.8600 - val_recall: 0.8352 - val_precision: 0.8850 - val_AUROC: 0.9875 - val_AUPRC: 0.9338 Epoch 18/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.4051 - accuracy: 0.8507 - recall: 0.8225 - precision: 0.8788 - AUROC: 0.9883 - AUPRC: 0.9320 Epoch 18: val_loss improved from 0.40520 to 0.39420, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 28s 16ms/step - loss: 0.4050 - accuracy: 0.8508 - recall: 0.8226 - precision: 0.8788 - AUROC: 0.9883 - AUPRC: 0.9320 - val_loss: 0.3942 - val_accuracy: 0.8612 - val_recall: 0.8418 - val_precision: 0.8815 - val_AUROC: 0.9883 - val_AUPRC: 0.9368 Epoch 19/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3985 - accuracy: 0.8532 - recall: 0.8267 - precision: 0.8788 - AUROC: 0.9886 - AUPRC: 0.9342 Epoch 19: val_loss did not improve from 0.39420 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3985 - accuracy: 0.8532 - recall: 0.8267 - precision: 0.8788 - AUROC: 0.9886 - AUPRC: 0.9342 - val_loss: 0.4075 - val_accuracy: 0.8581 - val_recall: 0.8374 - val_precision: 0.8787 - val_AUROC: 0.9872 - val_AUPRC: 0.9323 Epoch 20/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.3862 - accuracy: 0.8566 - recall: 0.8302 - precision: 0.8842 - AUROC: 0.9893 - AUPRC: 0.9374 Epoch 20: val_loss did not improve from 0.39420 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3864 - accuracy: 0.8566 - recall: 0.8302 - precision: 0.8842 - AUROC: 0.9893 - AUPRC: 0.9374 - val_loss: 0.4019 - val_accuracy: 0.8575 - val_recall: 0.8404 - val_precision: 0.8784 - val_AUROC: 0.9874 - val_AUPRC: 0.9344 Epoch 21/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.3898 - accuracy: 0.8558 - recall: 0.8308 - precision: 0.8823 - AUROC: 0.9890 - AUPRC: 0.9366 Epoch 21: val_loss improved from 0.39420 to 0.38808, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3897 - accuracy: 0.8558 - recall: 0.8308 - precision: 0.8824 - AUROC: 0.9890 - AUPRC: 0.9367 - val_loss: 0.3881 - val_accuracy: 0.8641 - val_recall: 0.8408 - val_precision: 0.8853 - val_AUROC: 0.9886 - val_AUPRC: 0.9379 Epoch 22/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3774 - accuracy: 0.8605 - recall: 0.8344 - precision: 0.8855 - AUROC: 0.9897 - AUPRC: 0.9399 Epoch 22: val_loss did not improve from 0.38808 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3774 - accuracy: 0.8605 - recall: 0.8345 - precision: 0.8855 - AUROC: 0.9896 - AUPRC: 0.9399 - val_loss: 0.4052 - val_accuracy: 0.8602 - val_recall: 0.8444 - val_precision: 0.8767 - val_AUROC: 0.9871 - val_AUPRC: 0.9350 Epoch 23/100 1800/1800 [==============================] - ETA: 0s - loss: 0.3706 - accuracy: 0.8635 - recall: 0.8385 - precision: 0.8890 - AUROC: 0.9900 - AUPRC: 0.9420 Epoch 23: val_loss did not improve from 0.38808 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3706 - accuracy: 0.8635 - recall: 0.8385 - precision: 0.8890 - AUROC: 0.9900 - AUPRC: 0.9420 - val_loss: 0.3886 - val_accuracy: 0.8645 - val_recall: 0.8468 - val_precision: 0.8827 - val_AUROC: 0.9884 - val_AUPRC: 0.9374 Epoch 24/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3709 - accuracy: 0.8625 - recall: 0.8379 - precision: 0.8865 - AUROC: 0.9900 - AUPRC: 0.9416 Epoch 24: val_loss did not improve from 0.38808 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3709 - accuracy: 0.8625 - recall: 0.8379 - precision: 0.8865 - AUROC: 0.9900 - AUPRC: 0.9416 - val_loss: 0.3901 - val_accuracy: 0.8644 - val_recall: 0.8431 - val_precision: 0.8830 - val_AUROC: 0.9881 - val_AUPRC: 0.9365 Epoch 25/100 1796/1800 [============================>.] - ETA: 0s - loss: 0.3634 - accuracy: 0.8660 - recall: 0.8422 - precision: 0.8903 - AUROC: 0.9903 - AUPRC: 0.9437 Epoch 25: val_loss improved from 0.38808 to 0.38364, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3634 - accuracy: 0.8660 - recall: 0.8422 - precision: 0.8903 - AUROC: 0.9903 - AUPRC: 0.9437 - val_loss: 0.3836 - val_accuracy: 0.8675 - val_recall: 0.8473 - val_precision: 0.8884 - val_AUROC: 0.9882 - val_AUPRC: 0.9388 Epoch 26/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.3544 - accuracy: 0.8686 - recall: 0.8448 - precision: 0.8904 - AUROC: 0.9908 - AUPRC: 0.9458 Epoch 26: val_loss improved from 0.38364 to 0.38140, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3544 - accuracy: 0.8686 - recall: 0.8448 - precision: 0.8904 - AUROC: 0.9908 - AUPRC: 0.9458 - val_loss: 0.3814 - val_accuracy: 0.8675 - val_recall: 0.8495 - val_precision: 0.8892 - val_AUROC: 0.9886 - val_AUPRC: 0.9399 Epoch 27/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3528 - accuracy: 0.8678 - recall: 0.8462 - precision: 0.8902 - AUROC: 0.9909 - AUPRC: 0.9464 Epoch 27: val_loss improved from 0.38140 to 0.37158, saving model to training_vgg_gray\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3528 - accuracy: 0.8678 - recall: 0.8461 - precision: 0.8902 - AUROC: 0.9909 - AUPRC: 0.9464 - val_loss: 0.3716 - val_accuracy: 0.8745 - val_recall: 0.8521 - val_precision: 0.8945 - val_AUROC: 0.9892 - val_AUPRC: 0.9422 Epoch 28/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.3500 - accuracy: 0.8702 - recall: 0.8472 - precision: 0.8926 - AUROC: 0.9909 - AUPRC: 0.9470 Epoch 28: val_loss did not improve from 0.37158 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3500 - accuracy: 0.8702 - recall: 0.8472 - precision: 0.8926 - AUROC: 0.9909 - AUPRC: 0.9470 - val_loss: 0.3835 - val_accuracy: 0.8686 - val_recall: 0.8516 - val_precision: 0.8889 - val_AUROC: 0.9883 - val_AUPRC: 0.9397 Epoch 29/100 1796/1800 [============================>.] - ETA: 0s - loss: 0.3456 - accuracy: 0.8712 - recall: 0.8489 - precision: 0.8924 - AUROC: 0.9911 - AUPRC: 0.9482 Epoch 29: val_loss did not improve from 0.37158 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3458 - accuracy: 0.8711 - recall: 0.8488 - precision: 0.8923 - AUROC: 0.9911 - AUPRC: 0.9482 - val_loss: 0.3834 - val_accuracy: 0.8687 - val_recall: 0.8519 - val_precision: 0.8865 - val_AUROC: 0.9881 - val_AUPRC: 0.9403 Epoch 30/100 1800/1800 [==============================] - ETA: 0s - loss: 0.3431 - accuracy: 0.8730 - recall: 0.8517 - precision: 0.8942 - AUROC: 0.9911 - AUPRC: 0.9487 Epoch 30: val_loss did not improve from 0.37158 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3431 - accuracy: 0.8730 - recall: 0.8517 - precision: 0.8942 - AUROC: 0.9911 - AUPRC: 0.9487 - val_loss: 0.3840 - val_accuracy: 0.8706 - val_recall: 0.8577 - val_precision: 0.8867 - val_AUROC: 0.9879 - val_AUPRC: 0.9399 Epoch 31/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.3372 - accuracy: 0.8749 - recall: 0.8537 - precision: 0.8950 - AUROC: 0.9915 - AUPRC: 0.9503 Epoch 31: val_loss did not improve from 0.37158 1800/1800 [==============================] - 27s 15ms/step - loss: 0.3370 - accuracy: 0.8750 - recall: 0.8538 - precision: 0.8950 - AUROC: 0.9915 - AUPRC: 0.9503 - val_loss: 0.3810 - val_accuracy: 0.8704 - val_recall: 0.8531 - val_precision: 0.8877 - val_AUROC: 0.9882 - val_AUPRC: 0.9409 338/338 [==============================] - 6s 14ms/step - loss: 0.3809 - accuracy: 0.8704 - recall: 0.8531 - precision: 0.8877 - AUROC: 0.9882 - AUPRC: 0.9409
1it [15:33, 933.03s/it]
In [44]:
print(f"VGG16 Soccer CNN Gray Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate: Accuracy : test - 0.8703703880310059 AUROC : test - 0.9881564378738403 AUPRC : test - 0.940942108631134 --------------------------------------------------------------------------------
In [45]:
# CNN VGG16 - Gray images - Metrics
CNN_metrics
Out[45]:
[{'test_evaluation': {'loss': 0.3809435963630676,
'accuracy': 0.8703703880310059,
'recall': 0.8530555367469788,
'precision': 0.8876577615737915,
'AUROC': 0.9881564378738403,
'AUPRC': 0.940942108631134}}]
In [75]:
# Save results (save dictionary to vgg_gray_metrics.pkl file)
with open('results/vgg_gray_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [46]:
# Save Gray VGG16 Soccer CNN
vgg16_soccer_gray = CNN_models[0]
vgg16_soccer_gray.save_weights('modelWeights/vgg16_soccer_gray_model.h5')
In [39]:
checkpoint_path = "training_vgg_rgb/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
save_best_only=True,
verbose=1)
interrupt = False
In [ ]:
# Flag to load weights if training got interrupted
interrupt = True
In [40]:
print("---- VGG16 Soccer CNN RGB ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(rgb_input, labels_soccer))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([rgb_input[x]/255 for x in train_indices]), np.array([rgb_input[x]/255 for x in test_indices])
y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]
# Get model
model = vgg16_soccer_model
# Load weights if training got interrupted
if interrupt:
model.load_weights(checkpoint_path)
CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
model,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size,
cp_callback
)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(model)
---- VGG16 Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1 Epoch 1/100 1800/1800 [==============================] - ETA: 0s - loss: 0.8185 - accuracy: 0.7068 - recall: 0.5898 - precision: 0.8062 - AUROC: 0.9581 - AUPRC: 0.7899 Epoch 1: val_loss improved from inf to 0.56561, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 39s 18ms/step - loss: 0.8185 - accuracy: 0.7068 - recall: 0.5898 - precision: 0.8062 - AUROC: 0.9581 - AUPRC: 0.7899 - val_loss: 0.5656 - val_accuracy: 0.8043 - val_recall: 0.7478 - val_precision: 0.8513 - val_AUROC: 0.9789 - val_AUPRC: 0.8845 Epoch 2/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.5878 - accuracy: 0.7893 - recall: 0.7309 - precision: 0.8428 - AUROC: 0.9772 - AUPRC: 0.8750 Epoch 2: val_loss improved from 0.56561 to 0.49451, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5876 - accuracy: 0.7894 - recall: 0.7310 - precision: 0.8429 - AUROC: 0.9772 - AUPRC: 0.8750 - val_loss: 0.4945 - val_accuracy: 0.8237 - val_recall: 0.7775 - val_precision: 0.8638 - val_AUROC: 0.9837 - val_AUPRC: 0.9077 Epoch 3/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.5184 - accuracy: 0.8132 - recall: 0.7677 - precision: 0.8546 - AUROC: 0.9818 - AUPRC: 0.8981 Epoch 3: val_loss improved from 0.49451 to 0.45372, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.5183 - accuracy: 0.8131 - recall: 0.7676 - precision: 0.8546 - AUROC: 0.9818 - AUPRC: 0.8981 - val_loss: 0.4537 - val_accuracy: 0.8427 - val_recall: 0.8031 - val_precision: 0.8800 - val_AUROC: 0.9861 - val_AUPRC: 0.9212 Epoch 4/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4797 - accuracy: 0.8278 - recall: 0.7871 - precision: 0.8645 - AUROC: 0.9841 - AUPRC: 0.9107 Epoch 4: val_loss improved from 0.45372 to 0.42020, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4797 - accuracy: 0.8278 - recall: 0.7871 - precision: 0.8645 - AUROC: 0.9841 - AUPRC: 0.9107 - val_loss: 0.4202 - val_accuracy: 0.8569 - val_recall: 0.8223 - val_precision: 0.8848 - val_AUROC: 0.9874 - val_AUPRC: 0.9295 Epoch 5/100 1800/1800 [==============================] - ETA: 0s - loss: 0.4474 - accuracy: 0.8407 - recall: 0.8044 - precision: 0.8733 - AUROC: 0.9859 - AUPRC: 0.9204 Epoch 5: val_loss improved from 0.42020 to 0.41564, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4474 - accuracy: 0.8407 - recall: 0.8044 - precision: 0.8733 - AUROC: 0.9859 - AUPRC: 0.9204 - val_loss: 0.4156 - val_accuracy: 0.8539 - val_recall: 0.8215 - val_precision: 0.8832 - val_AUROC: 0.9876 - val_AUPRC: 0.9310 Epoch 6/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.4222 - accuracy: 0.8481 - recall: 0.8178 - precision: 0.8769 - AUROC: 0.9872 - AUPRC: 0.9279 Epoch 6: val_loss improved from 0.41564 to 0.39419, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4220 - accuracy: 0.8483 - recall: 0.8179 - precision: 0.8770 - AUROC: 0.9873 - AUPRC: 0.9280 - val_loss: 0.3942 - val_accuracy: 0.8612 - val_recall: 0.8350 - val_precision: 0.8862 - val_AUROC: 0.9885 - val_AUPRC: 0.9358 Epoch 7/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.4052 - accuracy: 0.8546 - recall: 0.8247 - precision: 0.8826 - AUROC: 0.9882 - AUPRC: 0.9331 Epoch 7: val_loss improved from 0.39419 to 0.37756, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.4053 - accuracy: 0.8546 - recall: 0.8248 - precision: 0.8826 - AUROC: 0.9882 - AUPRC: 0.9331 - val_loss: 0.3776 - val_accuracy: 0.8670 - val_recall: 0.8440 - val_precision: 0.8894 - val_AUROC: 0.9889 - val_AUPRC: 0.9405 Epoch 8/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.3840 - accuracy: 0.8632 - recall: 0.8360 - precision: 0.8884 - AUROC: 0.9893 - AUPRC: 0.9385 Epoch 8: val_loss did not improve from 0.37756 1800/1800 [==============================] - 28s 16ms/step - loss: 0.3840 - accuracy: 0.8632 - recall: 0.8360 - precision: 0.8884 - AUROC: 0.9893 - AUPRC: 0.9385 - val_loss: 0.3780 - val_accuracy: 0.8709 - val_recall: 0.8540 - val_precision: 0.8867 - val_AUROC: 0.9887 - val_AUPRC: 0.9414 Epoch 9/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3720 - accuracy: 0.8665 - recall: 0.8402 - precision: 0.8911 - AUROC: 0.9899 - AUPRC: 0.9418 Epoch 9: val_loss improved from 0.37756 to 0.36596, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3719 - accuracy: 0.8665 - recall: 0.8402 - precision: 0.8911 - AUROC: 0.9900 - AUPRC: 0.9418 - val_loss: 0.3660 - val_accuracy: 0.8723 - val_recall: 0.8544 - val_precision: 0.8916 - val_AUROC: 0.9890 - val_AUPRC: 0.9442 Epoch 10/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.3568 - accuracy: 0.8708 - recall: 0.8474 - precision: 0.8949 - AUROC: 0.9906 - AUPRC: 0.9460 Epoch 10: val_loss improved from 0.36596 to 0.36201, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3568 - accuracy: 0.8708 - recall: 0.8473 - precision: 0.8948 - AUROC: 0.9906 - AUPRC: 0.9460 - val_loss: 0.3620 - val_accuracy: 0.8752 - val_recall: 0.8544 - val_precision: 0.8937 - val_AUROC: 0.9896 - val_AUPRC: 0.9453 Epoch 11/100 1800/1800 [==============================] - ETA: 0s - loss: 0.3467 - accuracy: 0.8724 - recall: 0.8498 - precision: 0.8948 - AUROC: 0.9911 - AUPRC: 0.9486 Epoch 11: val_loss improved from 0.36201 to 0.34957, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3467 - accuracy: 0.8724 - recall: 0.8498 - precision: 0.8948 - AUROC: 0.9911 - AUPRC: 0.9486 - val_loss: 0.3496 - val_accuracy: 0.8825 - val_recall: 0.8661 - val_precision: 0.9003 - val_AUROC: 0.9898 - val_AUPRC: 0.9486 Epoch 12/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3339 - accuracy: 0.8782 - recall: 0.8575 - precision: 0.8991 - AUROC: 0.9916 - AUPRC: 0.9520 Epoch 12: val_loss improved from 0.34957 to 0.33201, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3342 - accuracy: 0.8781 - recall: 0.8574 - precision: 0.8990 - AUROC: 0.9916 - AUPRC: 0.9519 - val_loss: 0.3320 - val_accuracy: 0.8885 - val_recall: 0.8724 - val_precision: 0.9050 - val_AUROC: 0.9907 - val_AUPRC: 0.9530 Epoch 13/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.3248 - accuracy: 0.8831 - recall: 0.8628 - precision: 0.9026 - AUROC: 0.9919 - AUPRC: 0.9540 Epoch 13: val_loss did not improve from 0.33201 1800/1800 [==============================] - 28s 15ms/step - loss: 0.3248 - accuracy: 0.8831 - recall: 0.8628 - precision: 0.9026 - AUROC: 0.9919 - AUPRC: 0.9540 - val_loss: 0.3371 - val_accuracy: 0.8865 - val_recall: 0.8724 - val_precision: 0.9038 - val_AUROC: 0.9902 - val_AUPRC: 0.9515 Epoch 14/100 1800/1800 [==============================] - ETA: 0s - loss: 0.3153 - accuracy: 0.8874 - recall: 0.8681 - precision: 0.9053 - AUROC: 0.9924 - AUPRC: 0.9563 Epoch 14: val_loss improved from 0.33201 to 0.33186, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3153 - accuracy: 0.8874 - recall: 0.8681 - precision: 0.9053 - AUROC: 0.9924 - AUPRC: 0.9563 - val_loss: 0.3319 - val_accuracy: 0.8884 - val_recall: 0.8741 - val_precision: 0.9037 - val_AUROC: 0.9906 - val_AUPRC: 0.9536 Epoch 15/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.3100 - accuracy: 0.8870 - recall: 0.8692 - precision: 0.9053 - AUROC: 0.9926 - AUPRC: 0.9575 Epoch 15: val_loss improved from 0.33186 to 0.32336, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.3100 - accuracy: 0.8870 - recall: 0.8693 - precision: 0.9053 - AUROC: 0.9926 - AUPRC: 0.9575 - val_loss: 0.3234 - val_accuracy: 0.8921 - val_recall: 0.8755 - val_precision: 0.9081 - val_AUROC: 0.9911 - val_AUPRC: 0.9553 Epoch 16/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.2988 - accuracy: 0.8904 - recall: 0.8735 - precision: 0.9082 - AUROC: 0.9931 - AUPRC: 0.9601 Epoch 16: val_loss did not improve from 0.32336 1800/1800 [==============================] - 28s 16ms/step - loss: 0.2989 - accuracy: 0.8904 - recall: 0.8735 - precision: 0.9082 - AUROC: 0.9931 - AUPRC: 0.9601 - val_loss: 0.3297 - val_accuracy: 0.8908 - val_recall: 0.8779 - val_precision: 0.9071 - val_AUROC: 0.9905 - val_AUPRC: 0.9538 Epoch 17/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.2987 - accuracy: 0.8921 - recall: 0.8761 - precision: 0.9098 - AUROC: 0.9930 - AUPRC: 0.9602 Epoch 17: val_loss did not improve from 0.32336 1800/1800 [==============================] - 28s 16ms/step - loss: 0.2987 - accuracy: 0.8922 - recall: 0.8762 - precision: 0.9099 - AUROC: 0.9930 - AUPRC: 0.9602 - val_loss: 0.3260 - val_accuracy: 0.8898 - val_recall: 0.8732 - val_precision: 0.9079 - val_AUROC: 0.9908 - val_AUPRC: 0.9548 Epoch 18/100 1796/1800 [============================>.] - ETA: 0s - loss: 0.2893 - accuracy: 0.8942 - recall: 0.8784 - precision: 0.9116 - AUROC: 0.9935 - AUPRC: 0.9623 Epoch 18: val_loss did not improve from 0.32336 1800/1800 [==============================] - 28s 15ms/step - loss: 0.2894 - accuracy: 0.8942 - recall: 0.8784 - precision: 0.9116 - AUROC: 0.9935 - AUPRC: 0.9622 - val_loss: 0.3235 - val_accuracy: 0.8931 - val_recall: 0.8778 - val_precision: 0.9084 - val_AUROC: 0.9909 - val_AUPRC: 0.9552 Epoch 19/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.2791 - accuracy: 0.8986 - recall: 0.8828 - precision: 0.9155 - AUROC: 0.9938 - AUPRC: 0.9644 Epoch 19: val_loss improved from 0.32336 to 0.31182, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.2790 - accuracy: 0.8986 - recall: 0.8828 - precision: 0.9155 - AUROC: 0.9938 - AUPRC: 0.9644 - val_loss: 0.3118 - val_accuracy: 0.8946 - val_recall: 0.8830 - val_precision: 0.9072 - val_AUROC: 0.9915 - val_AUPRC: 0.9583 Epoch 20/100 1798/1800 [============================>.] - ETA: 0s - loss: 0.2747 - accuracy: 0.8985 - recall: 0.8835 - precision: 0.9137 - AUROC: 0.9940 - AUPRC: 0.9654 Epoch 20: val_loss improved from 0.31182 to 0.31141, saving model to training_vgg_rgb\cp.ckpt 1800/1800 [==============================] - 29s 16ms/step - loss: 0.2749 - accuracy: 0.8984 - recall: 0.8834 - precision: 0.9136 - AUROC: 0.9940 - AUPRC: 0.9654 - val_loss: 0.3114 - val_accuracy: 0.8994 - val_recall: 0.8875 - val_precision: 0.9109 - val_AUROC: 0.9913 - val_AUPRC: 0.9587 Epoch 21/100 1797/1800 [============================>.] - ETA: 0s - loss: 0.2683 - accuracy: 0.9025 - recall: 0.8876 - precision: 0.9176 - AUROC: 0.9942 - AUPRC: 0.9667 Epoch 21: val_loss did not improve from 0.31141 1800/1800 [==============================] - 28s 15ms/step - loss: 0.2684 - accuracy: 0.9024 - recall: 0.8875 - precision: 0.9175 - AUROC: 0.9942 - AUPRC: 0.9667 - val_loss: 0.3253 - val_accuracy: 0.8940 - val_recall: 0.8844 - val_precision: 0.9060 - val_AUROC: 0.9905 - val_AUPRC: 0.9561 Epoch 22/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.2645 - accuracy: 0.9023 - recall: 0.8883 - precision: 0.9169 - AUROC: 0.9945 - AUPRC: 0.9675 Epoch 22: val_loss did not improve from 0.31141 1800/1800 [==============================] - 28s 15ms/step - loss: 0.2644 - accuracy: 0.9023 - recall: 0.8883 - precision: 0.9169 - AUROC: 0.9945 - AUPRC: 0.9675 - val_loss: 0.3222 - val_accuracy: 0.8943 - val_recall: 0.8831 - val_precision: 0.9070 - val_AUROC: 0.9909 - val_AUPRC: 0.9568 Epoch 23/100 1796/1800 [============================>.] - ETA: 0s - loss: 0.2631 - accuracy: 0.9034 - recall: 0.8899 - precision: 0.9177 - AUROC: 0.9944 - AUPRC: 0.9676 Epoch 23: val_loss did not improve from 0.31141 1800/1800 [==============================] - 28s 15ms/step - loss: 0.2631 - accuracy: 0.9034 - recall: 0.8899 - precision: 0.9177 - AUROC: 0.9944 - AUPRC: 0.9676 - val_loss: 0.3253 - val_accuracy: 0.8947 - val_recall: 0.8878 - val_precision: 0.9069 - val_AUROC: 0.9906 - val_AUPRC: 0.9564 Epoch 24/100 1799/1800 [============================>.] - ETA: 0s - loss: 0.2589 - accuracy: 0.9043 - recall: 0.8915 - precision: 0.9175 - AUROC: 0.9946 - AUPRC: 0.9687 Epoch 24: val_loss did not improve from 0.31141 1800/1800 [==============================] - 28s 15ms/step - loss: 0.2588 - accuracy: 0.9043 - recall: 0.8916 - precision: 0.9175 - AUROC: 0.9946 - AUPRC: 0.9687 - val_loss: 0.3252 - val_accuracy: 0.9011 - val_recall: 0.8919 - val_precision: 0.9120 - val_AUROC: 0.9900 - val_AUPRC: 0.9564 338/338 [==============================] - 6s 14ms/step - loss: 0.3252 - accuracy: 0.9011 - recall: 0.8919 - precision: 0.9120 - AUROC: 0.9900 - AUPRC: 0.9564
1it [12:32, 752.18s/it]
In [42]:
print(f"VGG16 Soccer CNN RGB Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate: Accuracy : test - 0.9011111259460449 AUROC : test - 0.990019679069519 AUPRC : test - 0.9564046263694763 --------------------------------------------------------------------------------
In [43]:
# CNN VGG16 - RGB Images - Metrics
CNN_metrics
Out[43]:
[{'test_evaluation': {'loss': 0.3251974582672119,
'accuracy': 0.9011111259460449,
'recall': 0.8919444680213928,
'precision': 0.9120431542396545,
'AUROC': 0.990019679069519,
'AUPRC': 0.9564046263694763}}]
In [77]:
# Save results (save dictionary to vgg_rgb_metrics.pkl file)
with open('results/vgg_rgb_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
In [44]:
# Save RGB VGG16 Soccer CNN
vgg16_soccer_rgb = CNN_models[0]
vgg16_soccer_rgb.save_weights('modelWeights/vgg16_soccer_rgb_model.h5')
Evaluation of models on the "Real Soccer" dataset¶
Evaluation on images not from the football analysis dataset to evaluate the generalization capabilities of the models on different kinds of soccer images.
In [37]:
image_shape_gray = (soccer_images_dim,soccer_images_dim,1)
In [53]:
## Load Models
oneDim_models = []
threeDim_models = []
# Create models
soccer = build_soccer_CNN(image_shape_gray)
soccer_data_augmentation = build_soccer_CNN(image_shape_gray)
soccer_resNet_model = resNet_soccer_model
soccer_vgg_model = vgg16_soccer_model
# Restore the weights
soccer.load_weights('modelWeights/soccerCNN_Model.h5')
soccer._name = "Basic_CNN"
soccer_data_augmentation.load_weights('modelWeights/soccerModelDataAugmentation.h5')
soccer_data_augmentation._name = "CNN_Data_Augmentation"
soccer_resNet_model.load_weights('modelWeights/resNet_soccer_gray_model.h5')
soccer_resNet_model._name = "ResNet_Soccer_Gray"
soccer_vgg_model.load_weights('modelWeights/vgg16_soccer_gray_model.h5')
soccer_vgg_model._name = "VGG16_Soccer_Gray"
oneDim_models.append(soccer)
oneDim_models.append(soccer_data_augmentation)
threeDim_models.append(soccer_resNet_model)
threeDim_models.append(soccer_vgg_model)
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_16 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_16 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_17 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_17 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_18 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_18 (MaxPoolin (None, 10, 10, 256) 0
g2D)
conv2d_19 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_19 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_5 (Flatten) (None, 6400) 0
dense_17 (Dense) (None, 128) 819328
dropout_11 (Dropout) (None, 128) 0
dense_18 (Dense) (None, 128) 16512
dropout_12 (Dropout) (None, 128) 0
dense_19 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
Model: "Soccer_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_20 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_20 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_21 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_21 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_22 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_22 (MaxPoolin (None, 10, 10, 256) 0
g2D)
conv2d_23 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_23 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_6 (Flatten) (None, 6400) 0
dense_20 (Dense) (None, 128) 819328
dropout_13 (Dropout) (None, 128) 0
dense_21 (Dense) (None, 128) 16512
dropout_14 (Dropout) (None, 128) 0
dense_22 (Dense) (None, 9) 1161
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
In [46]:
from sklearn.metrics import roc_curve, auc, roc_auc_score, precision_recall_curve, average_precision_score
from sklearn.preprocessing import label_binarize
def get_metrics(labels, predictions):
""" Returns the AUROC and AUPRC metrics given the true labels and the model predictions.
Parameters
------------------------
labels: tuple
The true labels (n_samples,).
predictions: np.array
The model predictions (n_samples,).
Returns
------------------------
accuracy: float
The accuracy metric.
auroc: float
The AUROC metric.
auprc: float
The AUPRC metric."""
# Change shape from (n_samples,) to (n_samples, n_classes)
labels = label_binarize(labels, classes=np.unique(labels))
predictions = label_binarize(predictions, classes=np.unique(predictions))
# Accuracy
accuracy = accuracy_score(labels, predictions)
# AUROC
auroc = roc_auc_score(labels, predictions)
# AUPRC
auprc = average_precision_score(labels, predictions)
return accuracy, auroc, auprc
In [47]:
def one_hot_max(arr):
""" Returns the index of maximum value in the array (prediction). """
max_index = np.argmax(arr)
one_hot_arr = np.zeros_like(arr)
one_hot_arr[max_index] = 1
return one_hot_arr
In [48]:
# Dataset preparation
real_data_soccer_all = []
for event in real_data_soccer:
for element in real_data_soccer[event]:
real_data_soccer_all.append(element)
print(f"Data shape: {np.array(real_data_soccer_all).shape}")
print(f"Labels shape: {real_labels_soccer.shape}")
# 3 Channel data for resNet and VGG16
threeChannels_real_input = np.array([cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in real_data_soccer_all])
threeChannels_real_input = np.array([x/255 for x in threeChannels_real_input])
Data shape: (90, 80, 80) Labels shape: (90, 9)
In [49]:
real_data_soccer_all = np.array([np.expand_dims(x/255, axis=-1) for x in real_data_soccer_all])
In [44]:
for model in oneDim_models:
real_data_soccer_predictions = get_predictions(model, real_data_soccer_all, 2)
# Shape (n_samples,)
labels = np.argmax(real_labels_soccer, axis=1)
predictions = np.argmax(real_data_soccer_predictions, axis=1)
acc, auroc, auprc = get_metrics(
labels = labels,
predictions = predictions
)
print(f"{model.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)
true_labels = labels
predicted_labels = predictions
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=9,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(8,8)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11")
plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11" )
plt.show()
for model in threeDim_models:
real_data_soccer_predictions = get_predictions(model, threeChannels_real_input, 2)
# Shape (n_samples,)
labels = labels
predictions = np.argmax(real_data_soccer_predictions, axis=1)
acc, auroc, auprc = get_metrics(
labels = labels,
predictions = predictions
)
print(f"{model.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)
true_labels = labels
predicted_labels = predictions
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=9,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(8,8)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11")
plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11" )
plt.show()
45/45 [==============================] - 0s 6ms/step Basic_CNN metrics on real dataset: Accuracy: 0.6888888888888889 ------------------------------ AUROC: 0.825 ------------------------------ AUPRC: 0.5385870302536969 --------------------------------------------------------------------------------
45/45 [==============================] - 0s 1ms/step CNN_Data_Augmentation metrics on real dataset: Accuracy: 0.7 ------------------------------ AUROC: 0.83125 ------------------------------ AUPRC: 0.572998366013072 --------------------------------------------------------------------------------
45/45 [==============================] - 0s 8ms/step ResNet_Soccer_Gray metrics on real dataset: Accuracy: 0.5444444444444444 ------------------------------ AUROC: 0.74375 ------------------------------ AUPRC: 0.39875423958757295 --------------------------------------------------------------------------------
45/45 [==============================] - 0s 3ms/step VGG16_Soccer_Gray metrics on real dataset: Accuracy: 0.7 ------------------------------ AUROC: 0.8312500000000002 ------------------------------ AUPRC: 0.5530253696920363 --------------------------------------------------------------------------------
Card classification (Red / Yellow)¶
Images previously classified as Cards are to be classified in Red vs Yellow.
The cards classifier is a CNN taking in input RGB images to exploit the required color information to perform the task.
In [103]:
folderCards = r'D:\MachineLearning\Datasets\Soccer\TrainTestCards'
data_cards, labels_cards = loadData(folderCards, True)
In [104]:
print("[n.images, (width, height, depth)]")
for folder in (data_cards):
print(f"--- {folder} ---")
print(f"shape: {len(data_cards[folder])} {data_cards[folder][1].shape}")
card_sample = data_cards['Red-Cards'][22]
print("-"*30)
print(f"label {labels_cards[22]}")
plt.subplot(1,2,1),plt.imshow(card_sample, cmap='gray', vmin=0, vmax=255)
[n.images, (width, height, depth)] --- Red-Cards --- shape: 6000 (80, 80, 3) --- Yellow-Cards --- shape: 6000 (80, 80, 3) ------------------------------ label Red-Cards
Out[104]:
(<Axes: >, <matplotlib.image.AxesImage at 0x161fa5e7cd0>)
In [105]:
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
labels_cards = pd.DataFrame(le.fit_transform(labels_cards))
print(labels_cards.sample(3))
0 8539 1 9951 1 4806 0
In [106]:
# data_cards is a dict containing ['events'...] so group together classes to evaluate
data_cards_all = []
for event in data_cards:
for element in data_cards[event]:
data_cards_all.append(element)
print(f"Data shape: {np.array(data_cards_all).shape}")
print(f"Labels shape: {labels_cards.shape}")
Data shape: (12000, 80, 80, 3) Labels shape: (12000, 1)
In [107]:
def build_cards_CNN(
input_shape: tuple
) -> tf.keras.Model:
""" Returns the Cards CNN model.
Parameters
------------------------
input_shape: tuple
The shape of the input.
Returns
------------------------
model: model
The Cards model."""
CNN = Sequential(name="cards_CNN")
CNN.add(layers.Input(input_shape))
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Flatten())
for i in range(2):
CNN.add(layers.Dense(units = 128, activation='relu'))
CNN.add(layers.Dropout(0.5))
CNN.add(layers.Dense(2, activation='sigmoid'))
CNN.compile(
optimizer='adam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
CNN.summary()
return CNN
Main training loop - Cards CNN¶
- Normalize RGB images' values in 0-1 range.
- Build and train the model over the training set and test it over the test set for the different holdouts.
In [109]:
number_of_splits = 2
holdouts_generator = StratifiedShuffleSplit(
n_splits = number_of_splits,
test_size = 0.2,
random_state = 42
)
In [110]:
epochs = 100
batch_size = 32
In [111]:
print("---- CARDS CNN ----")
CNN_cards_metrics = []
CNN_cards_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_cards_all, labels_cards))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([data_cards_all[x]/255 for x in train_indices]), np.array([data_cards_all[x]/255 for x in test_indices])
y_train, y_test = labels_cards.iloc[train_indices], labels_cards.iloc[test_indices]
# One hot encoding
y_train = one_hot_encoding(y_train, 2)
y_test = one_hot_encoding(y_test, 2)
# Build CNN
CNN = build_cards_CNN(x_train[0].shape)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_model(
CNN,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size
)
predictions = get_predictions(CNN, x_test, batch_size=batch_size)
CNN_holdout_predictions.append(predictions)
CNN_holdout_test_labels.append(y_test)
CNN_cards_metrics.append(CNN_holdout_metrics)
CNN_cards_history.append(CNN_holdout_history)
CNN_models.append(CNN)
---- CARDS CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "cards_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_24 (Conv2D) (None, 80, 80, 128) 3584
max_pooling2d_24 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_25 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_25 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_26 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_26 (MaxPoolin (None, 10, 10, 256) 0
g2D)
flatten_7 (Flatten) (None, 25600) 0
dense_23 (Dense) (None, 128) 3276928
dropout_15 (Dropout) (None, 128) 0
dense_24 (Dense) (None, 128) 16512
dropout_16 (Dropout) (None, 128) 0
dense_25 (Dense) (None, 2) 258
=================================================================
Total params: 3,740,034
Trainable params: 3,740,034
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
300/300 [==============================] - 8s 19ms/step - loss: 0.5340 - accuracy: 0.7060 - recall: 0.7081 - precision: 0.7045 - AUROC: 0.8004 - AUPRC: 0.8048 - f1_score: 0.7063 - balanced_accuracy: 0.7055 - specificity: 0.7029 - miss_rate: 0.2919 - fall_out: 0.2971 - mcc: 0.4110 - val_loss: 0.2701 - val_accuracy: 0.8917 - val_recall: 0.8917 - val_precision: 0.8920 - val_AUROC: 0.9557 - val_AUPRC: 0.9553 - val_f1_score: 0.8919 - val_balanced_accuracy: 0.8919 - val_specificity: 0.8921 - val_miss_rate: 0.1083 - val_fall_out: 0.1079 - val_mcc: 0.7838
Epoch 2/100
300/300 [==============================] - 5s 17ms/step - loss: 0.2000 - accuracy: 0.9285 - recall: 0.9275 - precision: 0.9288 - AUROC: 0.9760 - AUPRC: 0.9750 - f1_score: 0.9281 - balanced_accuracy: 0.9282 - specificity: 0.9289 - miss_rate: 0.0725 - fall_out: 0.0711 - mcc: 0.8564 - val_loss: 0.1520 - val_accuracy: 0.9367 - val_recall: 0.9379 - val_precision: 0.9360 - val_AUROC: 0.9858 - val_AUPRC: 0.9860 - val_f1_score: 0.9369 - val_balanced_accuracy: 0.9369 - val_specificity: 0.9358 - val_miss_rate: 0.0621 - val_fall_out: 0.0642 - val_mcc: 0.8738
Epoch 3/100
300/300 [==============================] - 5s 17ms/step - loss: 0.1253 - accuracy: 0.9543 - recall: 0.9542 - precision: 0.9544 - AUROC: 0.9899 - AUPRC: 0.9894 - f1_score: 0.9543 - balanced_accuracy: 0.9543 - specificity: 0.9544 - miss_rate: 0.0458 - fall_out: 0.0456 - mcc: 0.9085 - val_loss: 0.2360 - val_accuracy: 0.9267 - val_recall: 0.9279 - val_precision: 0.9271 - val_AUROC: 0.9762 - val_AUPRC: 0.9728 - val_f1_score: 0.9275 - val_balanced_accuracy: 0.9275 - val_specificity: 0.9271 - val_miss_rate: 0.0721 - val_fall_out: 0.0729 - val_mcc: 0.8550
Epoch 4/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0964 - accuracy: 0.9658 - recall: 0.9656 - precision: 0.9654 - AUROC: 0.9939 - AUPRC: 0.9937 - f1_score: 0.9655 - balanced_accuracy: 0.9655 - specificity: 0.9654 - miss_rate: 0.0344 - fall_out: 0.0346 - mcc: 0.9310 - val_loss: 0.1289 - val_accuracy: 0.9529 - val_recall: 0.9529 - val_precision: 0.9533 - val_AUROC: 0.9901 - val_AUPRC: 0.9898 - val_f1_score: 0.9531 - val_balanced_accuracy: 0.9531 - val_specificity: 0.9533 - val_miss_rate: 0.0471 - val_fall_out: 0.0467 - val_mcc: 0.9063
Epoch 5/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0696 - accuracy: 0.9761 - recall: 0.9757 - precision: 0.9763 - AUROC: 0.9966 - AUPRC: 0.9964 - f1_score: 0.9760 - balanced_accuracy: 0.9760 - specificity: 0.9764 - miss_rate: 0.0243 - fall_out: 0.0236 - mcc: 0.9521 - val_loss: 0.1453 - val_accuracy: 0.9467 - val_recall: 0.9463 - val_precision: 0.9463 - val_AUROC: 0.9877 - val_AUPRC: 0.9865 - val_f1_score: 0.9463 - val_balanced_accuracy: 0.9463 - val_specificity: 0.9463 - val_miss_rate: 0.0538 - val_fall_out: 0.0538 - val_mcc: 0.8925
Epoch 6/100
300/300 [==============================] - 5s 16ms/step - loss: 0.0610 - accuracy: 0.9772 - recall: 0.9770 - precision: 0.9773 - AUROC: 0.9974 - AUPRC: 0.9974 - f1_score: 0.9771 - balanced_accuracy: 0.9771 - specificity: 0.9773 - miss_rate: 0.0230 - fall_out: 0.0227 - mcc: 0.9543 - val_loss: 0.1364 - val_accuracy: 0.9571 - val_recall: 0.9579 - val_precision: 0.9571 - val_AUROC: 0.9906 - val_AUPRC: 0.9890 - val_f1_score: 0.9575 - val_balanced_accuracy: 0.9575 - val_specificity: 0.9571 - val_miss_rate: 0.0421 - val_fall_out: 0.0429 - val_mcc: 0.9150
300/300 [==============================] - 2s 6ms/step - loss: 0.0381 - accuracy: 0.9864 - recall: 0.9860 - precision: 0.9861 - AUROC: 0.9987 - AUPRC: 0.9985 - f1_score: 0.9861 - balanced_accuracy: 0.9861 - specificity: 0.9861 - miss_rate: 0.0140 - fall_out: 0.0139 - mcc: 0.9722
75/75 [==============================] - 0s 6ms/step - loss: 0.1364 - accuracy: 0.9571 - recall: 0.9579 - precision: 0.9571 - AUROC: 0.9906 - AUPRC: 0.9890 - f1_score: 0.9575 - balanced_accuracy: 0.9575 - specificity: 0.9571 - miss_rate: 0.0421 - fall_out: 0.0429 - mcc: 0.9150
75/75 [==============================] - 0s 4ms/step
1it [00:38, 38.92s/it]
-- HOLDOUT 2
Model: "cards_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_27 (Conv2D) (None, 80, 80, 128) 3584
max_pooling2d_27 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_28 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_28 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_29 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_29 (MaxPoolin (None, 10, 10, 256) 0
g2D)
flatten_8 (Flatten) (None, 25600) 0
dense_26 (Dense) (None, 128) 3276928
dropout_17 (Dropout) (None, 128) 0
dense_27 (Dense) (None, 128) 16512
dropout_18 (Dropout) (None, 128) 0
dense_28 (Dense) (None, 2) 258
=================================================================
Total params: 3,740,034
Trainable params: 3,740,034
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
300/300 [==============================] - 8s 20ms/step - loss: 0.4929 - accuracy: 0.7400 - recall: 0.7268 - precision: 0.7414 - AUROC: 0.8344 - AUPRC: 0.8392 - f1_score: 0.7340 - balanced_accuracy: 0.7367 - specificity: 0.7466 - miss_rate: 0.2732 - fall_out: 0.2534 - mcc: 0.4734 - val_loss: 0.2235 - val_accuracy: 0.9158 - val_recall: 0.9146 - val_precision: 0.9161 - val_AUROC: 0.9698 - val_AUPRC: 0.9694 - val_f1_score: 0.9153 - val_balanced_accuracy: 0.9154 - val_specificity: 0.9162 - val_miss_rate: 0.0854 - val_fall_out: 0.0838 - val_mcc: 0.8308
Epoch 2/100
300/300 [==============================] - 5s 17ms/step - loss: 0.1735 - accuracy: 0.9357 - recall: 0.9370 - precision: 0.9358 - AUROC: 0.9815 - AUPRC: 0.9809 - f1_score: 0.9364 - balanced_accuracy: 0.9364 - specificity: 0.9357 - miss_rate: 0.0630 - fall_out: 0.0643 - mcc: 0.8727 - val_loss: 0.1530 - val_accuracy: 0.9471 - val_recall: 0.9479 - val_precision: 0.9471 - val_AUROC: 0.9854 - val_AUPRC: 0.9845 - val_f1_score: 0.9475 - val_balanced_accuracy: 0.9475 - val_specificity: 0.9471 - val_miss_rate: 0.0521 - val_fall_out: 0.0529 - val_mcc: 0.8950
Epoch 3/100
300/300 [==============================] - 5s 18ms/step - loss: 0.1166 - accuracy: 0.9579 - recall: 0.9577 - precision: 0.9576 - AUROC: 0.9913 - AUPRC: 0.9911 - f1_score: 0.9577 - balanced_accuracy: 0.9577 - specificity: 0.9576 - miss_rate: 0.0423 - fall_out: 0.0424 - mcc: 0.9153 - val_loss: 0.1239 - val_accuracy: 0.9592 - val_recall: 0.9588 - val_precision: 0.9595 - val_AUROC: 0.9900 - val_AUPRC: 0.9889 - val_f1_score: 0.9591 - val_balanced_accuracy: 0.9592 - val_specificity: 0.9596 - val_miss_rate: 0.0413 - val_fall_out: 0.0404 - val_mcc: 0.9183
Epoch 4/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0861 - accuracy: 0.9702 - recall: 0.9696 - precision: 0.9697 - AUROC: 0.9950 - AUPRC: 0.9948 - f1_score: 0.9696 - balanced_accuracy: 0.9696 - specificity: 0.9697 - miss_rate: 0.0304 - fall_out: 0.0303 - mcc: 0.9393 - val_loss: 0.1796 - val_accuracy: 0.9442 - val_recall: 0.9446 - val_precision: 0.9434 - val_AUROC: 0.9842 - val_AUPRC: 0.9824 - val_f1_score: 0.9440 - val_balanced_accuracy: 0.9440 - val_specificity: 0.9433 - val_miss_rate: 0.0554 - val_fall_out: 0.0567 - val_mcc: 0.8879
Epoch 5/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0736 - accuracy: 0.9709 - recall: 0.9712 - precision: 0.9710 - AUROC: 0.9963 - AUPRC: 0.9961 - f1_score: 0.9711 - balanced_accuracy: 0.9711 - specificity: 0.9710 - miss_rate: 0.0288 - fall_out: 0.0290 - mcc: 0.9423 - val_loss: 0.1295 - val_accuracy: 0.9546 - val_recall: 0.9550 - val_precision: 0.9538 - val_AUROC: 0.9900 - val_AUPRC: 0.9891 - val_f1_score: 0.9544 - val_balanced_accuracy: 0.9544 - val_specificity: 0.9538 - val_miss_rate: 0.0450 - val_fall_out: 0.0463 - val_mcc: 0.9088
300/300 [==============================] - 2s 6ms/step - loss: 0.0500 - accuracy: 0.9847 - recall: 0.9847 - precision: 0.9845 - AUROC: 0.9987 - AUPRC: 0.9987 - f1_score: 0.9846 - balanced_accuracy: 0.9846 - specificity: 0.9845 - miss_rate: 0.0153 - fall_out: 0.0155 - mcc: 0.9692
75/75 [==============================] - 0s 6ms/step - loss: 0.1295 - accuracy: 0.9546 - recall: 0.9550 - precision: 0.9538 - AUROC: 0.9900 - AUPRC: 0.9891 - f1_score: 0.9544 - balanced_accuracy: 0.9544 - specificity: 0.9538 - miss_rate: 0.0450 - fall_out: 0.0463 - mcc: 0.9088
75/75 [==============================] - 0s 4ms/step
2it [01:16, 38.09s/it]
In [112]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_cards_metrics, number_of_splits)
print(f"Red VS Yellow Card - CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_cards_history)
print("-"*100)
Red VS Yellow Card - CNN Metrics - 2-holdouts estimate: Accuracy : train - 0.9855208396911621 -- test - 0.9558333456516266 AUROC : train - 0.9987120628356934 -- test - 0.990309476852417 AUPRC : train - 0.9986125230789185 -- test - 0.9890577793121338 -------------------------------------------------------------------------------- CNN - Train history:
----------------------------------------------------------------------------------------------------
In [113]:
# Save Cards CNN
cards_model = CNN_models[0]
cards_model.save_weights('modelWeights/cards_model.h5')
In [114]:
# Cards CNN Metrics
CNN_cards_metrics[0]
Out[114]:
{'train_evaluation': {'loss': 0.03812405839562416,
'accuracy': 0.9863541722297668,
'recall': 0.9860416650772095,
'precision': 0.9861443638801575,
'AUROC': 0.9986873865127563,
'AUPRC': 0.9985128045082092,
'f1_score': 0.9860930442810059,
'balanced_accuracy': 0.9860937595367432,
'specificity': 0.9861458539962769,
'miss_rate': 0.013958333060145378,
'fall_out': 0.013854166492819786,
'mcc': 0.9721875190734863},
'test_evaluation': {'loss': 0.13638216257095337,
'accuracy': 0.9570833444595337,
'recall': 0.9579166769981384,
'precision': 0.9571190476417542,
'AUROC': 0.9905983209609985,
'AUPRC': 0.9889681339263916,
'f1_score': 0.957517683506012,
'balanced_accuracy': 0.9574999809265137,
'specificity': 0.9570833444595337,
'miss_rate': 0.04208333417773247,
'fall_out': 0.042916666716337204,
'mcc': 0.915000319480896}}
Confusion Matrix of Cards CNN¶
In [116]:
holdout_to_plot = 0
true_labels = tf.math.argmax(CNN_holdout_test_labels[holdout_to_plot], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_predictions[holdout_to_plot], axis=1)
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=2,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set(rc = {'figure.figsize':(5,5)})
plt.xlabel('Predicted Labels', fontsize=10)
plt.ylabel('True Labels', fontsize=10)
plt.yticks(np.arange(2)+0.5,('Red','Yellow'), fontsize="10")
plt.xticks(np.arange(2)+0.5,('Red','Yellow'), fontsize="10" )
plt.show()
Generic soccer images classification¶
To distinguish between generic soccer images and soccer images belonging to a particular event a CNN is designed.
The model takes as input event and generic soccer images and considers the generic soccer event as an additional class.
In [24]:
epochs = 100
batch_size = 32
In [25]:
#data_soccer is a dict containing ['events'...] so group together classes to evaluate
data_soccer_with_generic_all = []
for event in data_soccer:
for element in data_soccer[event]:
data_soccer_with_generic_all.append(element)
for element in data_generic_events["Soccer"]:
data_soccer_with_generic_all.append(element)
print(f"Data shape: {np.array(data_soccer_with_generic_all).shape}")
print(f"Labels shape: {labels_soccer_with_generic.shape}")
Data shape: (55199, 80, 80) Labels shape: (55199, 10)
In [26]:
data_soccer_with_generic_all = [np.expand_dims(x, axis=-1) for x in data_soccer_with_generic_all]
In [27]:
def build_soccer_CNN_with_generic(
input_shape: tuple
) -> tf.keras.Model:
""" Returns the generic soccer CNN model.
Parameters
------------------------
input_shape: tuple
The shape of the input.
Returns
------------------------
model: model
The generic soccer model."""
CNN = Sequential(name="Soccer_with_generic_CNN")
CNN.add(layers.Input(input_shape))
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Flatten())
for i in range(2):
CNN.add(layers.Dense(units = 128, activation='relu'))
CNN.add(layers.Dropout(0.5))
CNN.add(layers.Dense(10, activation='softmax'))
CNN.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=get_minimal_multiclass_metrics()
)
CNN.summary()
return CNN
Main training loop - Soccer CNN + generic event¶
- Normalize images' values in 0-1 range.
- Build and train the model over the training set and test it over the test set for the different holdouts.
In [68]:
print("---- Soccer+generic CNN ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_with_generic_all, labels_soccer_with_generic))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([data_soccer_with_generic_all[x]/255 for x in train_indices]), np.array([data_soccer_with_generic_all[x]/255 for x in test_indices])
y_train, y_test = labels_soccer_with_generic.iloc[train_indices], labels_soccer_with_generic.iloc[test_indices]
# Build CNN
CNN = build_soccer_CNN_with_generic(x_train[0].shape)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_model(
CNN,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size
)
predictions = get_predictions(CNN, x_test, batch_size=batch_size)
CNN_holdout_predictions.append(predictions)
CNN_holdout_test_labels.append(y_test)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(CNN)
---- Soccer+generic CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_12 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_12 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_13 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_13 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_14 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_14 (MaxPoolin (None, 10, 10, 256) 0
g2D)
conv2d_15 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_15 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_3 (Flatten) (None, 6400) 0
dense_9 (Dense) (None, 128) 819328
dropout_6 (Dropout) (None, 128) 0
dense_10 (Dense) (None, 128) 16512
dropout_7 (Dropout) (None, 128) 0
dense_11 (Dense) (None, 10) 1290
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
1380/1380 [==============================] - 26s 18ms/step - loss: 1.3459 - accuracy: 0.4946 - recall: 0.3038 - precision: 0.7262 - AUROC: 0.8963 - AUPRC: 0.5650 - val_loss: 0.6595 - val_accuracy: 0.7771 - val_recall: 0.6816 - val_precision: 0.8659 - val_AUROC: 0.9761 - val_AUPRC: 0.8600
Epoch 2/100
1380/1380 [==============================] - 24s 17ms/step - loss: 0.7398 - accuracy: 0.7548 - recall: 0.6603 - precision: 0.8371 - AUROC: 0.9679 - AUPRC: 0.8323 - val_loss: 0.4211 - val_accuracy: 0.8646 - val_recall: 0.8136 - val_precision: 0.9103 - val_AUROC: 0.9896 - val_AUPRC: 0.9368
Epoch 3/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.5390 - accuracy: 0.8249 - recall: 0.7695 - precision: 0.8783 - AUROC: 0.9815 - AUPRC: 0.9021 - val_loss: 0.3836 - val_accuracy: 0.8788 - val_recall: 0.8344 - val_precision: 0.9200 - val_AUROC: 0.9908 - val_AUPRC: 0.9469
Epoch 4/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.4291 - accuracy: 0.8608 - recall: 0.8230 - precision: 0.9015 - AUROC: 0.9874 - AUPRC: 0.9333 - val_loss: 0.2878 - val_accuracy: 0.9064 - val_recall: 0.8878 - val_precision: 0.9291 - val_AUROC: 0.9944 - val_AUPRC: 0.9661
Epoch 5/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3716 - accuracy: 0.8809 - recall: 0.8529 - precision: 0.9134 - AUROC: 0.9900 - AUPRC: 0.9479 - val_loss: 0.3053 - val_accuracy: 0.9010 - val_recall: 0.8820 - val_precision: 0.9232 - val_AUROC: 0.9930 - val_AUPRC: 0.9624
Epoch 6/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3212 - accuracy: 0.8989 - recall: 0.8751 - precision: 0.9262 - AUROC: 0.9920 - AUPRC: 0.9589 - val_loss: 0.2412 - val_accuracy: 0.9195 - val_recall: 0.9055 - val_precision: 0.9356 - val_AUROC: 0.9951 - val_AUPRC: 0.9744
Epoch 7/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2869 - accuracy: 0.9090 - recall: 0.8898 - precision: 0.9333 - AUROC: 0.9933 - AUPRC: 0.9663 - val_loss: 0.2319 - val_accuracy: 0.9250 - val_recall: 0.9089 - val_precision: 0.9414 - val_AUROC: 0.9953 - val_AUPRC: 0.9761
Epoch 8/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2553 - accuracy: 0.9187 - recall: 0.9019 - precision: 0.9399 - AUROC: 0.9944 - AUPRC: 0.9723 - val_loss: 0.2204 - val_accuracy: 0.9289 - val_recall: 0.9183 - val_precision: 0.9405 - val_AUROC: 0.9950 - val_AUPRC: 0.9776
Epoch 9/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2392 - accuracy: 0.9240 - recall: 0.9091 - precision: 0.9435 - AUROC: 0.9948 - AUPRC: 0.9748 - val_loss: 0.1992 - val_accuracy: 0.9349 - val_recall: 0.9253 - val_precision: 0.9496 - val_AUROC: 0.9961 - val_AUPRC: 0.9809
Epoch 10/100
1380/1380 [==============================] - 26s 19ms/step - loss: 0.2084 - accuracy: 0.9332 - recall: 0.9201 - precision: 0.9492 - AUROC: 0.9959 - AUPRC: 0.9800 - val_loss: 0.2128 - val_accuracy: 0.9364 - val_recall: 0.9292 - val_precision: 0.9449 - val_AUROC: 0.9946 - val_AUPRC: 0.9776
Epoch 11/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.1975 - accuracy: 0.9375 - recall: 0.9254 - precision: 0.9530 - AUROC: 0.9960 - AUPRC: 0.9815 - val_loss: 0.2290 - val_accuracy: 0.9334 - val_recall: 0.9265 - val_precision: 0.9416 - val_AUROC: 0.9936 - val_AUPRC: 0.9748
345/345 [==============================] - 2s 6ms/step - loss: 0.2290 - accuracy: 0.9334 - recall: 0.9265 - precision: 0.9416 - AUROC: 0.9936 - AUPRC: 0.9748
345/345 [==============================] - 2s 4ms/step
1it [04:49, 289.95s/it]
-- HOLDOUT 2
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_16 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_16 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_17 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_17 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_18 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_18 (MaxPoolin (None, 10, 10, 256) 0
g2D)
conv2d_19 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_19 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_4 (Flatten) (None, 6400) 0
dense_12 (Dense) (None, 128) 819328
dropout_8 (Dropout) (None, 128) 0
dense_13 (Dense) (None, 128) 16512
dropout_9 (Dropout) (None, 128) 0
dense_14 (Dense) (None, 10) 1290
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
1380/1380 [==============================] - 27s 19ms/step - loss: 1.3777 - accuracy: 0.4855 - recall: 0.2880 - precision: 0.7169 - AUROC: 0.8912 - AUPRC: 0.5474 - val_loss: 0.7099 - val_accuracy: 0.7647 - val_recall: 0.6688 - val_precision: 0.8466 - val_AUROC: 0.9722 - val_AUPRC: 0.8428
Epoch 2/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.7676 - accuracy: 0.7400 - recall: 0.6429 - precision: 0.8255 - AUROC: 0.9659 - AUPRC: 0.8200 - val_loss: 0.4444 - val_accuracy: 0.8515 - val_recall: 0.8079 - val_precision: 0.9020 - val_AUROC: 0.9885 - val_AUPRC: 0.9326
Epoch 3/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.5745 - accuracy: 0.8123 - recall: 0.7530 - precision: 0.8703 - AUROC: 0.9794 - AUPRC: 0.8919 - val_loss: 0.3906 - val_accuracy: 0.8772 - val_recall: 0.8272 - val_precision: 0.9238 - val_AUROC: 0.9917 - val_AUPRC: 0.9489
Epoch 4/100
1380/1380 [==============================] - 26s 19ms/step - loss: 0.4593 - accuracy: 0.8507 - recall: 0.8098 - precision: 0.8952 - AUROC: 0.9859 - AUPRC: 0.9256 - val_loss: 0.3006 - val_accuracy: 0.9040 - val_recall: 0.8798 - val_precision: 0.9297 - val_AUROC: 0.9940 - val_AUPRC: 0.9643
Epoch 5/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3975 - accuracy: 0.8711 - recall: 0.8391 - precision: 0.9094 - AUROC: 0.9890 - AUPRC: 0.9424 - val_loss: 0.2791 - val_accuracy: 0.9138 - val_recall: 0.8895 - val_precision: 0.9348 - val_AUROC: 0.9942 - val_AUPRC: 0.9682
Epoch 6/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3414 - accuracy: 0.8899 - recall: 0.8635 - precision: 0.9200 - AUROC: 0.9914 - AUPRC: 0.9551 - val_loss: 0.2998 - val_accuracy: 0.9024 - val_recall: 0.8793 - val_precision: 0.9294 - val_AUROC: 0.9935 - val_AUPRC: 0.9655
Epoch 7/100
1380/1380 [==============================] - 31s 23ms/step - loss: 0.3008 - accuracy: 0.9036 - recall: 0.8820 - precision: 0.9320 - AUROC: 0.9930 - AUPRC: 0.9638 - val_loss: 0.2484 - val_accuracy: 0.9255 - val_recall: 0.9132 - val_precision: 0.9399 - val_AUROC: 0.9941 - val_AUPRC: 0.9720
Epoch 8/100
1380/1380 [==============================] - 44s 32ms/step - loss: 0.2664 - accuracy: 0.9151 - recall: 0.8959 - precision: 0.9385 - AUROC: 0.9942 - AUPRC: 0.9704 - val_loss: 0.2605 - val_accuracy: 0.9164 - val_recall: 0.8975 - val_precision: 0.9384 - val_AUROC: 0.9946 - val_AUPRC: 0.9713
Epoch 9/100
1380/1380 [==============================] - 45s 32ms/step - loss: 0.2508 - accuracy: 0.9184 - recall: 0.9009 - precision: 0.9401 - AUROC: 0.9945 - AUPRC: 0.9727 - val_loss: 0.2333 - val_accuracy: 0.9286 - val_recall: 0.9193 - val_precision: 0.9419 - val_AUROC: 0.9943 - val_AUPRC: 0.9756
Epoch 10/100
1380/1380 [==============================] - 45s 33ms/step - loss: 0.2262 - accuracy: 0.9268 - recall: 0.9118 - precision: 0.9458 - AUROC: 0.9954 - AUPRC: 0.9773 - val_loss: 0.2288 - val_accuracy: 0.9299 - val_recall: 0.9201 - val_precision: 0.9428 - val_AUROC: 0.9943 - val_AUPRC: 0.9761
Epoch 11/100
1380/1380 [==============================] - 45s 33ms/step - loss: 0.2143 - accuracy: 0.9320 - recall: 0.9173 - precision: 0.9504 - AUROC: 0.9955 - AUPRC: 0.9789 - val_loss: 0.2264 - val_accuracy: 0.9313 - val_recall: 0.9234 - val_precision: 0.9439 - val_AUROC: 0.9945 - val_AUPRC: 0.9773
Epoch 12/100
1380/1380 [==============================] - 42s 30ms/step - loss: 0.1942 - accuracy: 0.9378 - recall: 0.9236 - precision: 0.9552 - AUROC: 0.9963 - AUPRC: 0.9821 - val_loss: 0.1948 - val_accuracy: 0.9439 - val_recall: 0.9358 - val_precision: 0.9529 - val_AUROC: 0.9950 - val_AUPRC: 0.9805
Epoch 13/100
1380/1380 [==============================] - 44s 32ms/step - loss: 0.1889 - accuracy: 0.9409 - recall: 0.9283 - precision: 0.9567 - AUROC: 0.9961 - AUPRC: 0.9828 - val_loss: 0.2340 - val_accuracy: 0.9294 - val_recall: 0.9204 - val_precision: 0.9412 - val_AUROC: 0.9937 - val_AUPRC: 0.9753
Epoch 14/100
1380/1380 [==============================] - 43s 31ms/step - loss: 0.1748 - accuracy: 0.9436 - recall: 0.9326 - precision: 0.9593 - AUROC: 0.9967 - AUPRC: 0.9851 - val_loss: 0.2211 - val_accuracy: 0.9364 - val_recall: 0.9294 - val_precision: 0.9471 - val_AUROC: 0.9938 - val_AUPRC: 0.9767
345/345 [==============================] - 3s 10ms/step - loss: 0.2211 - accuracy: 0.9364 - recall: 0.9294 - precision: 0.9471 - AUROC: 0.9938 - AUPRC: 0.9767
345/345 [==============================] - 2s 6ms/step
2it [13:13, 396.60s/it]
In [69]:
print(f"Soccer+Generic CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 2-holdouts estimate: Accuracy : test - 0.9334239363670349 AUROC : test - 0.9935887455940247 AUPRC : test - 0.9748353958129883 --------------------------------------------------------------------------------
Confusion Matrix - Soccer CNN + generic event¶
Note that the generic class has fewer examples and only 206 test samples.
In [75]:
true_labels = tf.math.argmax(CNN_holdout_predictions[1], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_test_labels[1], axis=1)
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=10,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(10,10)})
plt.xlabel('Predicted Labels', fontsize=14)
plt.ylabel('True Labels', fontsize=14)
plt.yticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11")
plt.xticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11" )
plt.show()
In [73]:
# Save Soccer CNN with generic class
soccerAndGeneric_model = CNN_models[1]
soccerAndGeneric_model.save_weights('modelWeights/soccerAndGeneric_model.h5')
In [72]:
# Soccer+generic CNN metrics
CNN_metrics[1]
Out[72]:
{'test_evaluation': {'loss': 0.22106623649597168,
'accuracy': 0.936413049697876,
'recall': 0.9294384121894836,
'precision': 0.9471109509468079,
'AUROC': 0.9938239455223083,
'AUPRC': 0.976667046546936}}
In [79]:
# Save results (save dictionary to soccer_with_generic_metrics.pkl file)
with open('results/soccer_with_generic_metrics.pkl', 'wb') as fp:
pickle.dump(CNN_metrics, fp)
print('dictionary saved successfully to file')
dictionary saved successfully to file
Evaluation on real dataset¶
In [46]:
image_shape_gray = (soccer_images_dim,soccer_images_dim,1)
image_shape_rgb = (soccer_images_dim,soccer_images_dim,3)
In [76]:
## Load Model
# Create models
soccer_events_and_generic = build_soccer_CNN_with_generic(image_shape_gray)
# Restore the weights
soccer_events_and_generic.load_weights('modelWeights/soccerAndGeneric_model.h5')
soccer_events_and_generic._name = "soccer_events_and_generic"
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_20 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_20 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_21 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_21 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_22 (Conv2D) (None, 20, 20, 256) 295168
max_pooling2d_22 (MaxPoolin (None, 10, 10, 256) 0
g2D)
conv2d_23 (Conv2D) (None, 10, 10, 256) 590080
max_pooling2d_23 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_5 (Flatten) (None, 6400) 0
dense_15 (Dense) (None, 128) 819328
dropout_10 (Dropout) (None, 128) 0
dense_16 (Dense) (None, 128) 16512
dropout_11 (Dropout) (None, 128) 0
dense_17 (Dense) (None, 10) 1290
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
In [61]:
folderMyTestDataset = r'D:\MachineLearning\Datasets\Soccer\myDataset' # RealSoccer + events and genericSoccer images
data_myTest_t, _ = loadData(folderMyTestDataset, False)
In [62]:
soccer_generic_data = []
for folder in data_myTest_t:
if folder != "Events" and folder != "Generic":
for i in range(len(data_myTest_t[folder])):
soccer_generic_data.append(data_myTest_t[folder][i])
for folder in data_myTest_t:
if folder == "Generic":
for i in range(len(data_myTest_t[folder])):
soccer_generic_data.append(data_myTest_t[folder][i])
In [63]:
soccer_generic_data = np.array([np.expand_dims(x/255, axis=-1) for x in soccer_generic_data])
In [64]:
print(f"Data shape: {soccer_generic_data.shape}")
print(f"Labels shape: {soccer_generic_labels.shape}")
Data shape: (100, 80, 80, 1) Labels shape: (100, 10)
In [77]:
soccer_generic_predictions = get_predictions(soccer_events_and_generic, soccer_generic_data, 2)
# Shape (n_samples,)
labels = np.argmax(soccer_generic_labels, axis=1)
predictions = np.argmax(soccer_generic_predictions, axis=1)
acc, auroc, auprc = get_metrics(
labels = labels,
predictions = predictions
)
print(f"{soccer_events_and_generic.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)
true_labels = labels
predicted_labels = predictions
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=10,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(8,8)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11")
plt.xticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11" )
plt.show()
50/50 [==============================] - 0s 5ms/step soccer_events_and_generic metrics on real dataset: Accuracy: 0.62 ------------------------------ AUROC: 0.7888888888888889 ------------------------------ AUPRC: 0.45661860361860357 --------------------------------------------------------------------------------
Complete classification pipeline definition¶
So far a few models were developed soccer event detection:
- Soccer CNN
- Soccer CNN with Data Augmentation
- ResNet
- VGG16
- Soccer CNN + generic event
The basic soccer CNN and soccer CNN with generic class show the best results under these specific test conditions.
A model for Cards binary classification was also deployed:
- Cards CNN
Here is a comparison of the various models performances on the event soccer detection task.
In [1]:
results = {"model":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
In [4]:
# Load Results (Read dictionary pkl file)
with open('results/basic_CNN_metrics.pkl', 'rb') as fp:
basic_CNN_metrics = pickle.load(fp)
with open('results/data_augmentation_basic_CNN_metrics.pkl', 'rb') as fp:
data_augmentation_basic_CNN_metrics = pickle.load(fp)
with open('results/resNet_gray_metrics.pkl', 'rb') as fp:
resNet_gray_metrics = pickle.load(fp)
with open('results/resNet_rgb_metrics.pkl', 'rb') as fp:
resNet_rgb_metrics = pickle.load(fp)
with open('results/vgg_gray_metrics.pkl', 'rb') as fp:
vgg_gray_metrics = pickle.load(fp)
with open('results/vgg_rgb_metrics.pkl', 'rb') as fp:
vgg_rgb_metrics = pickle.load(fp)
with open('results/soccer_with_generic_metrics.pkl', 'rb') as fp:
soccer_with_generic_metrics = []
soccer_with_generic_metrics.append(pickle.load(fp))
In [6]:
# CNN metrics
for metrics, model in ((basic_CNN_metrics, "basic CNN"), (data_augmentation_basic_CNN_metrics, "data augmentation basic CNN"), (resNet_gray_metrics, "resNet gray CNN"), (resNet_rgb_metrics, "resNet rgb CNN"), (vgg_gray_metrics, "vgg gray CNN"), (vgg_rgb_metrics, "vgg rgb CNN"), (soccer_with_generic_metrics, "soccer with generic CNN")):
i = 0
for holdout in metrics:
results["model"].append(model)
results["accuracy"].append(holdout["test_evaluation"]["accuracy"])
results["AUROC"].append(holdout["test_evaluation"]["AUROC"])
results["AUPRC"].append(holdout["test_evaluation"]["AUPRC"])
In [8]:
import pandas as pd
In [9]:
results_df = pd.DataFrame(results)
In [ ]:
#TODO make nice graph
In [17]:
from barplots import barplots
barplots(
results_df,
groupby=["model"],
orientation="horizontal",
height=12,
legend_position="upper left"
)
Out[17]:
[(<Figure size 2400x420 with 1 Axes>,
array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
(<Figure size 2400x420 with 1 Axes>,
array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object)),
(<Figure size 2400x420 with 1 Axes>,
array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
dtype=object))]
The next task to tackle is:
- Random image classification: filter out non soccer-related images.
This problem is studied with two approaches, generiting two different classification pipelines.
Approach A: Supervised Learning
Soccer VS Random classification CNN > Soccer Event Classification CNN > Cards Classification CNN.
Approach B: Unupervised Learning
Soccer VS Random clustering > Soccer Event Classification CNN > Cards Classification CNN.
Approach A - Supervised Learning¶
Design a CNN model for the binary classification on soccer related images and random images.
In [26]:
number_of_splits = 2
holdouts_generator = StratifiedShuffleSplit(
n_splits = number_of_splits,
test_size = 0.2,
random_state = 42
)
In [27]:
epochs = 100
batch_size = 32
In [29]:
# Create dataset (random soccer samples and event samples)
data_soccer_vs_events = []
labels_soccer_vs_events = []
# 1000 soccer images from soccer events
data_soccer_events = [np.squeeze(x, axis=-1) for x in data_soccer_all]
soccer_elements = random.sample(data_soccer_events, 1000)
data_soccer_vs_events.extend(soccer_elements)
# 400 soccer images from general soccer
general_soccer_elements = random.sample(data_generic_events["Soccer"], 400)
data_soccer_vs_events.extend(general_soccer_elements)
# 1400 images from generic events
data_soccer_vs_events.extend(data_generic_events["Random"])
for i in range(1400):
labels_soccer_vs_events.append(0)
for i in range(1400):
labels_soccer_vs_events.append(1)
labels_soccer_vs_events = one_hot_encoding(labels_soccer_vs_events, 2)
print(f"Data shape: {np.array(data_soccer_vs_events).shape}")
print(f"Labels shape: {labels_soccer_vs_events.shape}")
Data shape: (2800, 80, 80) Labels shape: (2800, 2)
In [30]:
data_soccer_vs_events = [np.expand_dims(x, axis=-1) for x in data_soccer_vs_events]
In [31]:
def build_soccer_nonSoccer_CNN(
input_shape: tuple
) -> tf.keras.Model:
""" Returns the basic soccer CNN model.
Parameters
------------------------
input_shape: tuple
The shape of the input.
Returns
------------------------
model: model
The soccer vs random model."""
CNN = Sequential(name="Soccer_Events_CNN")
CNN.add(layers.Input(input_shape))
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 128,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Conv2D(
filters = 256,
kernel_size = 3,
padding='same',
activation='relu'))
CNN.add(layers.MaxPooling2D()),
CNN.add(layers.Flatten())
for i in range(2):
CNN.add(layers.Dense(units = 128, activation='relu'))
CNN.add(layers.Dropout(0.5))
CNN.add(layers.Dense(2, activation='softmax'))
CNN.compile(
optimizer='adam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
CNN.summary()
return CNN
Main training loop - Soccer vs Non-Soccer CNN¶
- Normalize images' values in 0-1 range.
- Build and train the model over the training set and test it over the test set for the different holdouts.
In [39]:
print("---- Soccer vs Non-Soccer CNN ----")
CNN_metrics = []
CNN_history = []
CNN_holdout_predictions = []
CNN_holdout_test_labels = []
CNN_models = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_vs_events, labels_soccer_vs_events))):
print(f"-- HOLDOUT {holdout_number+1}")
# Train/Test data + Normalization
x_train, x_test = np.array([data_soccer_vs_events[x]/255 for x in train_indices]), np.array([data_soccer_vs_events[x]/255 for x in test_indices])
y_train, y_test = labels_soccer_vs_events.iloc[train_indices], labels_soccer_vs_events.iloc[test_indices]
# Build CNN
CNN = build_soccer_nonSoccer_CNN(x_train[0].shape)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_model(
CNN,
x_train,
x_test,
y_train.values,
y_test.values,
epochs,
batch_size
)
predictions = get_predictions(CNN, x_test, batch_size=batch_size)
CNN_holdout_predictions.append(predictions)
CNN_holdout_test_labels.append(y_test)
CNN_metrics.append(CNN_holdout_metrics)
CNN_history.append(CNN_holdout_history)
CNN_models.append(CNN)
---- Soccer vs Non-Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_Events_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_16 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_16 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_17 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_17 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_18 (Conv2D) (None, 20, 20, 128) 147584
max_pooling2d_18 (MaxPoolin (None, 10, 10, 128) 0
g2D)
conv2d_19 (Conv2D) (None, 10, 10, 256) 295168
max_pooling2d_19 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_4 (Flatten) (None, 6400) 0
dense_12 (Dense) (None, 128) 819328
dropout_8 (Dropout) (None, 128) 0
dense_13 (Dense) (None, 128) 16512
dropout_9 (Dropout) (None, 128) 0
dense_14 (Dense) (None, 2) 258
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
70/70 [==============================] - 3s 22ms/step - loss: 0.6000 - accuracy: 0.6621 - recall: 0.6621 - precision: 0.6621 - AUROC: 0.7414 - AUPRC: 0.7439 - f1_score: 0.6621 - balanced_accuracy: 0.6621 - specificity: 0.6621 - miss_rate: 0.3379 - fall_out: 0.3379 - mcc: 0.3241 - val_loss: 0.3223 - val_accuracy: 0.8679 - val_recall: 0.8679 - val_precision: 0.8679 - val_AUROC: 0.9379 - val_AUPRC: 0.9351 - val_f1_score: 0.8679 - val_balanced_accuracy: 0.8679 - val_specificity: 0.8679 - val_miss_rate: 0.1321 - val_fall_out: 0.1321 - val_mcc: 0.7357
Epoch 2/100
70/70 [==============================] - 1s 16ms/step - loss: 0.2208 - accuracy: 0.9170 - recall: 0.9170 - precision: 0.9170 - AUROC: 0.9696 - AUPRC: 0.9657 - f1_score: 0.9170 - balanced_accuracy: 0.9170 - specificity: 0.9170 - miss_rate: 0.0830 - fall_out: 0.0830 - mcc: 0.8339 - val_loss: 0.1392 - val_accuracy: 0.9464 - val_recall: 0.9464 - val_precision: 0.9464 - val_AUROC: 0.9898 - val_AUPRC: 0.9893 - val_f1_score: 0.9464 - val_balanced_accuracy: 0.9464 - val_specificity: 0.9464 - val_miss_rate: 0.0536 - val_fall_out: 0.0536 - val_mcc: 0.8929
Epoch 3/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1640 - accuracy: 0.9415 - recall: 0.9415 - precision: 0.9415 - AUROC: 0.9819 - AUPRC: 0.9790 - f1_score: 0.9415 - balanced_accuracy: 0.9415 - specificity: 0.9415 - miss_rate: 0.0585 - fall_out: 0.0585 - mcc: 0.8830 - val_loss: 0.1067 - val_accuracy: 0.9607 - val_recall: 0.9607 - val_precision: 0.9607 - val_AUROC: 0.9926 - val_AUPRC: 0.9921 - val_f1_score: 0.9607 - val_balanced_accuracy: 0.9607 - val_specificity: 0.9607 - val_miss_rate: 0.0393 - val_fall_out: 0.0393 - val_mcc: 0.9214
Epoch 4/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1273 - accuracy: 0.9580 - recall: 0.9580 - precision: 0.9580 - AUROC: 0.9870 - AUPRC: 0.9844 - f1_score: 0.9580 - balanced_accuracy: 0.9580 - specificity: 0.9580 - miss_rate: 0.0420 - fall_out: 0.0420 - mcc: 0.9161 - val_loss: 0.1484 - val_accuracy: 0.9536 - val_recall: 0.9536 - val_precision: 0.9536 - val_AUROC: 0.9781 - val_AUPRC: 0.9727 - val_f1_score: 0.9536 - val_balanced_accuracy: 0.9536 - val_specificity: 0.9536 - val_miss_rate: 0.0464 - val_fall_out: 0.0464 - val_mcc: 0.9071
Epoch 5/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1297 - accuracy: 0.9527 - recall: 0.9527 - precision: 0.9527 - AUROC: 0.9848 - AUPRC: 0.9818 - f1_score: 0.9527 - balanced_accuracy: 0.9527 - specificity: 0.9527 - miss_rate: 0.0473 - fall_out: 0.0473 - mcc: 0.9054 - val_loss: 0.1183 - val_accuracy: 0.9696 - val_recall: 0.9696 - val_precision: 0.9696 - val_AUROC: 0.9817 - val_AUPRC: 0.9765 - val_f1_score: 0.9696 - val_balanced_accuracy: 0.9696 - val_specificity: 0.9696 - val_miss_rate: 0.0304 - val_fall_out: 0.0304 - val_mcc: 0.9393
70/70 [==============================] - 0s 6ms/step - loss: 0.0714 - accuracy: 0.9741 - recall: 0.9741 - precision: 0.9741 - AUROC: 0.9941 - AUPRC: 0.9929 - f1_score: 0.9741 - balanced_accuracy: 0.9741 - specificity: 0.9741 - miss_rate: 0.0259 - fall_out: 0.0259 - mcc: 0.9482
18/18 [==============================] - 0s 6ms/step - loss: 0.1183 - accuracy: 0.9696 - recall: 0.9696 - precision: 0.9696 - AUROC: 0.9817 - AUPRC: 0.9765 - f1_score: 0.9696 - balanced_accuracy: 0.9696 - specificity: 0.9696 - miss_rate: 0.0304 - fall_out: 0.0304 - mcc: 0.9393
18/18 [==============================] - 0s 4ms/step
1it [00:08, 8.43s/it]
-- HOLDOUT 2
Model: "Soccer_Events_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_20 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_20 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_21 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_21 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_22 (Conv2D) (None, 20, 20, 128) 147584
max_pooling2d_22 (MaxPoolin (None, 10, 10, 128) 0
g2D)
conv2d_23 (Conv2D) (None, 10, 10, 256) 295168
max_pooling2d_23 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_5 (Flatten) (None, 6400) 0
dense_15 (Dense) (None, 128) 819328
dropout_10 (Dropout) (None, 128) 0
dense_16 (Dense) (None, 128) 16512
dropout_11 (Dropout) (None, 128) 0
dense_17 (Dense) (None, 2) 258
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/100
70/70 [==============================] - 2s 22ms/step - loss: 0.5652 - accuracy: 0.6772 - recall: 0.6772 - precision: 0.6772 - AUROC: 0.7733 - AUPRC: 0.7796 - f1_score: 0.6772 - balanced_accuracy: 0.6772 - specificity: 0.6772 - miss_rate: 0.3228 - fall_out: 0.3228 - mcc: 0.3545 - val_loss: 0.1876 - val_accuracy: 0.9286 - val_recall: 0.9286 - val_precision: 0.9286 - val_AUROC: 0.9778 - val_AUPRC: 0.9763 - val_f1_score: 0.9286 - val_balanced_accuracy: 0.9286 - val_specificity: 0.9286 - val_miss_rate: 0.0714 - val_fall_out: 0.0714 - val_mcc: 0.8571
Epoch 2/100
70/70 [==============================] - 1s 16ms/step - loss: 0.2142 - accuracy: 0.9246 - recall: 0.9246 - precision: 0.9246 - AUROC: 0.9730 - AUPRC: 0.9700 - f1_score: 0.9246 - balanced_accuracy: 0.9246 - specificity: 0.9246 - miss_rate: 0.0754 - fall_out: 0.0754 - mcc: 0.8491 - val_loss: 0.1192 - val_accuracy: 0.9518 - val_recall: 0.9518 - val_precision: 0.9518 - val_AUROC: 0.9934 - val_AUPRC: 0.9936 - val_f1_score: 0.9518 - val_balanced_accuracy: 0.9518 - val_specificity: 0.9518 - val_miss_rate: 0.0482 - val_fall_out: 0.0482 - val_mcc: 0.9036
Epoch 3/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1478 - accuracy: 0.9509 - recall: 0.9509 - precision: 0.9509 - AUROC: 0.9840 - AUPRC: 0.9812 - f1_score: 0.9509 - balanced_accuracy: 0.9509 - specificity: 0.9509 - miss_rate: 0.0491 - fall_out: 0.0491 - mcc: 0.9018 - val_loss: 0.1293 - val_accuracy: 0.9571 - val_recall: 0.9571 - val_precision: 0.9571 - val_AUROC: 0.9764 - val_AUPRC: 0.9698 - val_f1_score: 0.9571 - val_balanced_accuracy: 0.9571 - val_specificity: 0.9571 - val_miss_rate: 0.0429 - val_fall_out: 0.0429 - val_mcc: 0.9143
Epoch 4/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1589 - accuracy: 0.9406 - recall: 0.9406 - precision: 0.9406 - AUROC: 0.9817 - AUPRC: 0.9788 - f1_score: 0.9406 - balanced_accuracy: 0.9406 - specificity: 0.9406 - miss_rate: 0.0594 - fall_out: 0.0594 - mcc: 0.8813 - val_loss: 0.1174 - val_accuracy: 0.9536 - val_recall: 0.9536 - val_precision: 0.9536 - val_AUROC: 0.9862 - val_AUPRC: 0.9838 - val_f1_score: 0.9536 - val_balanced_accuracy: 0.9536 - val_specificity: 0.9536 - val_miss_rate: 0.0464 - val_fall_out: 0.0464 - val_mcc: 0.9071
Epoch 5/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1201 - accuracy: 0.9621 - recall: 0.9621 - precision: 0.9621 - AUROC: 0.9865 - AUPRC: 0.9835 - f1_score: 0.9621 - balanced_accuracy: 0.9621 - specificity: 0.9621 - miss_rate: 0.0379 - fall_out: 0.0379 - mcc: 0.9241 - val_loss: 0.1691 - val_accuracy: 0.9214 - val_recall: 0.9214 - val_precision: 0.9214 - val_AUROC: 0.9790 - val_AUPRC: 0.9766 - val_f1_score: 0.9214 - val_balanced_accuracy: 0.9214 - val_specificity: 0.9214 - val_miss_rate: 0.0786 - val_fall_out: 0.0786 - val_mcc: 0.8429
Epoch 6/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1184 - accuracy: 0.9594 - recall: 0.9594 - precision: 0.9594 - AUROC: 0.9893 - AUPRC: 0.9875 - f1_score: 0.9594 - balanced_accuracy: 0.9594 - specificity: 0.9594 - miss_rate: 0.0406 - fall_out: 0.0406 - mcc: 0.9187 - val_loss: 0.1247 - val_accuracy: 0.9554 - val_recall: 0.9554 - val_precision: 0.9554 - val_AUROC: 0.9840 - val_AUPRC: 0.9805 - val_f1_score: 0.9554 - val_balanced_accuracy: 0.9554 - val_specificity: 0.9554 - val_miss_rate: 0.0446 - val_fall_out: 0.0446 - val_mcc: 0.9107
70/70 [==============================] - 0s 6ms/step - loss: 0.0956 - accuracy: 0.9670 - recall: 0.9670 - precision: 0.9670 - AUROC: 0.9893 - AUPRC: 0.9867 - f1_score: 0.9670 - balanced_accuracy: 0.9670 - specificity: 0.9670 - miss_rate: 0.0330 - fall_out: 0.0330 - mcc: 0.9339
18/18 [==============================] - 0s 6ms/step - loss: 0.1247 - accuracy: 0.9554 - recall: 0.9554 - precision: 0.9554 - AUROC: 0.9840 - AUPRC: 0.9805 - f1_score: 0.9554 - balanced_accuracy: 0.9554 - specificity: 0.9554 - miss_rate: 0.0446 - fall_out: 0.0446 - mcc: 0.9107
18/18 [==============================] - 0s 4ms/step
2it [00:17, 8.68s/it]
In [40]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)
print(f"Soccer VS Events - CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_history)
print("-"*100)
Soccer VS Events - CNN Metrics - 2-holdouts estimate: Accuracy : train - 0.9705357253551483 -- test - 0.9625000059604645 AUROC : train - 0.9916850328445435 -- test - 0.982855498790741 AUPRC : train - 0.9897907674312592 -- test - 0.9784724414348602 -------------------------------------------------------------------------------- CNN - Train history:
----------------------------------------------------------------------------------------------------
In [42]:
# Soccer vs Non-Soccer CNN Metrics
CNN_metrics[0]
Out[42]:
{'train_evaluation': {'loss': 0.0713760033249855,
'accuracy': 0.9741071462631226,
'recall': 0.9741071462631226,
'precision': 0.9741071462631226,
'AUROC': 0.9940677881240845,
'AUPRC': 0.9928805828094482,
'f1_score': 0.9741071462631226,
'balanced_accuracy': 0.9741071462631226,
'specificity': 0.9741071462631226,
'miss_rate': 0.02589285746216774,
'fall_out': 0.02589285746216774,
'mcc': 0.9482142925262451},
'test_evaluation': {'loss': 0.11834557354450226,
'accuracy': 0.9696428775787354,
'recall': 0.9696428775787354,
'precision': 0.9696428775787354,
'AUROC': 0.9816644787788391,
'AUPRC': 0.9764835834503174,
'f1_score': 0.9696428775787354,
'balanced_accuracy': 0.9696428775787354,
'specificity': 0.9696428775787354,
'miss_rate': 0.03035714291036129,
'fall_out': 0.03035714291036129,
'mcc': 0.9392856955528259}}
In [43]:
# Save Soccer CNN
soccer_nonSoccer_model = CNN_models[0]
soccer_nonSoccer_model.save_weights('modelWeights/soccer_nonSoccer_model.h5')
Evaluation on myDataset¶
In [47]:
## Load Model
# Create models
events_vs_soccer = build_soccer_nonSoccer_CNN(image_shape_gray)
# Restore the weights
events_vs_soccer.load_weights('modelWeights/events_vs_soccer_model.h5')
events_vs_soccer._name = "events_vs_soccer"
Model: "Soccer_Events_CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_24 (Conv2D) (None, 80, 80, 128) 1280
max_pooling2d_24 (MaxPoolin (None, 40, 40, 128) 0
g2D)
conv2d_25 (Conv2D) (None, 40, 40, 128) 147584
max_pooling2d_25 (MaxPoolin (None, 20, 20, 128) 0
g2D)
conv2d_26 (Conv2D) (None, 20, 20, 128) 147584
max_pooling2d_26 (MaxPoolin (None, 10, 10, 128) 0
g2D)
conv2d_27 (Conv2D) (None, 10, 10, 256) 295168
max_pooling2d_27 (MaxPoolin (None, 5, 5, 256) 0
g2D)
flatten_6 (Flatten) (None, 6400) 0
dense_18 (Dense) (None, 128) 819328
dropout_12 (Dropout) (None, 128) 0
dense_19 (Dense) (None, 128) 16512
dropout_13 (Dropout) (None, 128) 0
dense_20 (Dense) (None, 2) 258
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
In [48]:
#data_soccer is a dict containing ['events'...] so group together classes to evaluate
data_myTest_all = []
for event in data_myTest:
for element in data_myTest[event]:
data_myTest_all.append(element)
print(f"Data shape: {np.array(data_myTest_all).shape}")
print(f"Labels shape: {labels_myTest.shape}")
Data shape: (110, 80, 80) Labels shape: (110, 11)
In [49]:
data_myTest_all = np.array([np.expand_dims(x/255, axis=-1) for x in data_myTest_all])
In [50]:
# Adjust labels for each step (only for final evaluation and confrontation)
# Events VS Soccer
events_vs_soccer_labels = pd.DataFrame(columns=[0,1])
event_value = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
for index, row in labels_myTest.iterrows():
if np.array_equal(row.values, event_value):
new_row = {0: 0.0, 1: 1.0}
events_vs_soccer_labels.loc[index] = new_row
else:
new_row = {0: 1.0, 1: 0.0}
events_vs_soccer_labels.loc[index] = new_row
events_vs_soccer_labels.sample(2)
Out[50]:
| 0 | 1 | |
|---|---|---|
| 17 | 1.0 | 0.0 |
| 92 | 1.0 | 0.0 |
In [56]:
event_vs_soccer_predictions = get_predictions(events_vs_soccer, data_myTest_all, batch_size=2)
# Shape (n_samples,)
labels = np.argmax(events_vs_soccer_labels, axis=1)
predictions = np.argmax(event_vs_soccer_predictions, axis=1)
acc, auroc, auprc = get_metrics(
labels = labels,
predictions = predictions
)
print(f"{events_vs_soccer.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)
true_labels = labels
predicted_labels = predictions
confusion_matrix = tf.math.confusion_matrix(
true_labels,
predicted_labels,
num_classes=2,
weights=None,
name=None
)
sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(5,5)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(2)+0.5,('Soccer','Random'), fontsize="11")
plt.xticks(np.arange(2)+0.5,('Soccer','Random'), fontsize="11" )
plt.show()
55/55 [==============================] - 0s 1ms/step events_vs_soccer metrics on real dataset: Accuracy: 0.7181818181818181 ------------------------------ AUROC: 0.7999999999999999 ------------------------------ AUPRC: 0.2167832167832168 --------------------------------------------------------------------------------
Approach B - Unsupervised Learning¶
Evaluate the clustering algorithms of KMeans and Agglomerative Clustering for the binary task of distinguishing soccer related images and random images.
In [138]:
n_labels = np.argmax(labels_generic_events, axis=-1)
In [139]:
from sklearn.cluster import KMeans, AgglomerativeClustering
def clustering(
data: np.array,
n_clusters: int
):
"Return clustering predictions with Kmeans and Agglomerative Clustering."
clustering_predictions = {}
clustering_predictions["KMeans"] = []
clustering_predictions["Agglomerative"] = []
K_clust = KMeans(n_clusters = n_clusters, random_state = 42, n_init="auto").fit(data)
A_clust = AgglomerativeClustering(n_clusters = n_clusters).fit(data)
clustering_predictions["KMeans"].append(K_clust.fit_predict(data))
clustering_predictions["Agglomerative"].append(A_clust.fit_predict(data))
return clustering_predictions
In [140]:
from sklearn.decomposition import PCA
def get_pca_decomposition(
X: np.array
) -> pd.DataFrame:
"""Return the 2D PCA decomposition of the given data."""
return PCA(n_components=2).fit_transform(X)
In [141]:
# RGB images
folderGenericEvents = r'D:\MachineLearning\Datasets\Soccer\GenericEvents' # Generic soccer and random events
data_generic_events_rgb, labels_generic_events_rgb = loadData(folderGenericEvents, True)
unsupervised_input_data_rgb_all = []
for event in data_generic_events_rgb:
for element in data_generic_events_rgb[event]:
unsupervised_input_data_rgb_all.append(element)
unsupervised_input_data_rgb_all = np.array(unsupervised_input_data_rgb_all)
# Gray scale images
unsupervised_input_data_all = []
for event in data_generic_events:
for element in data_generic_events[event]:
unsupervised_input_data_all.append(element)
unsupervised_input_data_all = np.array(unsupervised_input_data_all)
In [142]:
print("-- Input data for clustering --")
# Raw pixel values
raw_input = unsupervised_input_data_all.reshape(2600, -1)
print(f"Raw pixel values: {raw_input.shape}")
# Raw pixel values (PCA -> 2)
PCA_raw_input = get_pca_decomposition(raw_input)
print(f"Raw pixel values (PCA -> 2): {PCA_raw_input.shape}")
# Raw pixel values RGB
raw_input_rgb = unsupervised_input_data_rgb_all.reshape(2600, -1)
print(f"Raw pixel values RGB: {raw_input_rgb.shape}")
# Raw pixel values (PCA -> 2)
PCA_raw_input_rgb = get_pca_decomposition(raw_input_rgb)
print(f"Raw pixel values RGB (PCA -> 2): {PCA_raw_input_rgb.shape}")
# Representation extracted from pre trained model
threeChannels_input = np.array([cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in unsupervised_input_data_all])
image_shape = threeChannels_input.shape[1:]
resNet_feature_extraction = tf.keras.applications.resnet.ResNet50(input_shape=image_shape,
include_top=False,
weights='imagenet')
resNet_feature_extraction.trainable = False
# Gray images
features = get_predictions(resNet_feature_extraction, threeChannels_input, 64).reshape(2600, -1)
print(f"CNN features: {features.shape}")
# RGB images
features_rgb = get_predictions(resNet_feature_extraction, unsupervised_input_data_rgb_all, 64).reshape(2600, -1)
print(f"CNN features RGB: {features_rgb.shape}")
# Representation extracted from pre trained model + avg pooling
inputs = tf.keras.Input(shape=image_shape)
x = resNet_feature_extraction(inputs)
outputs = layers.GlobalAveragePooling2D()(x)
resNet_feature_extraction_pool = tf.keras.Model(inputs, outputs)
# Gray images
features_pooling = get_predictions(resNet_feature_extraction_pool, threeChannels_input, 64).reshape(2600, -1)
print(f"CNN features + pooling: {features_pooling.shape}")
# RGB images
features_pooling_rgb = get_predictions(resNet_feature_extraction_pool, unsupervised_input_data_rgb_all, 64).reshape(2600, -1)
print(f"CNN features + pooling: {features_pooling_rgb.shape}")
# (PCA -> 2) features
PCA_features = get_pca_decomposition(features)
print(f"CNN features (PCA -> 2): {PCA_features.shape}")
PCA_features_pooling = get_pca_decomposition(features_pooling)
print(f"CNN features + pooling (PCA -> 2): {PCA_features_pooling.shape}")
PCA_features_rgb = get_pca_decomposition(features_rgb)
print(f"CNN features RGB (PCA -> 2): {PCA_features_rgb.shape}")
PCA_features_pooling_rgb = get_pca_decomposition(features_pooling_rgb)
print(f"CNN features + pooling RGB (PCA -> 2): {PCA_features_pooling_rgb.shape}")
import skimage.feature
matrix_features = []
hog_features = []
for image in unsupervised_input_data_all:
# Texture Features to capture texture patterns.
# Calculate the co-occurrence matrix for the image and extract texture features
co_matrix = skimage.feature.graycomatrix(image, [5], [0], levels=256)
image_matrix_features = []
contrast = skimage.feature.graycoprops(co_matrix, 'contrast')
correlation = skimage.feature.graycoprops(co_matrix, 'correlation')
energy = skimage.feature.graycoprops(co_matrix, 'energy')
homogeneity = skimage.feature.graycoprops(co_matrix, 'homogeneity')
image_matrix_features.append(contrast)
image_matrix_features.append(correlation)
image_matrix_features.append(energy)
image_matrix_features.append(homogeneity)
matrix_features.append(image_matrix_features)
# Hog Features to capture object shapes and structures.
from skimage.feature import hog
from skimage import data, exposure
hog_image = hog(
image,
orientations=5,
pixels_per_cell=(10, 10),
cells_per_block=(1, 1)
)
hog_features.append(hog_image)
matrix_features = np.array(matrix_features)
matrix_features = matrix_features.squeeze()
print(f"Co-occurrence matrix features {matrix_features.shape}")
PCA_matrix_features = get_pca_decomposition(matrix_features)
print(f"Co-occurrence matrix features (PCA -> 2) {PCA_matrix_features.shape}")
hog_features = np.array(hog_features)
print(f"HOG features {hog_features.shape}")
PCA_hog_features = get_pca_decomposition(hog_features)
print(f"HOG features (PCA -> 2) {PCA_hog_features.shape}")
-- Input data for clustering -- Raw pixel values: (2600, 6400) Raw pixel values (PCA -> 2): (2600, 2) Raw pixel values RGB: (2600, 19200) Raw pixel values RGB (PCA -> 2): (2600, 2) 41/41 [==============================] - 1s 21ms/step CNN features: (2600, 18432) 41/41 [==============================] - 1s 13ms/step CNN features RGB: (2600, 18432) 41/41 [==============================] - 1s 14ms/step CNN features + pooling: (2600, 2048) 41/41 [==============================] - 1s 14ms/step CNN features + pooling: (2600, 2048) CNN features (PCA -> 2): (2600, 2) CNN features + pooling (PCA -> 2): (2600, 2) CNN features RGB (PCA -> 2): (2600, 2) CNN features + pooling RGB (PCA -> 2): (2600, 2) Co-occurrence matrix features (2600, 4) Co-occurrence matrix features (PCA -> 2) (2600, 2) HOG features (2600, 320) HOG features (PCA -> 2) (2600, 2)
In [143]:
predictions = {}
for input_data_name, input_data in (
("Raw", raw_input),
("PCA_raw", PCA_raw_input),
("Raw_rgb", raw_input_rgb),
("PCA_raw_rgb", PCA_raw_input_rgb),
("Features", features),
("Features+Pooling", features_pooling),
("Features_rgb", features_rgb),
("Features+Pooling_rgb", features_pooling_rgb),
("PCA_features", PCA_features),
("PCA_features_pooling", PCA_features_pooling),
("PCA_features_rgb", PCA_features_rgb),
("PCA_features_pooling_rgb", PCA_features_pooling_rgb),
("Matrix_features", matrix_features),
("PCA_matrix_features", PCA_matrix_features),
("Hog_features", hog_features),
("PCA_hog_features", PCA_hog_features)):
predictions[input_data_name] = []
predictions[input_data_name].append(clustering(input_data, 2))
Labels mismatch¶
This is binary clustering and the clustering algorithms are given the n_clusters=2 and always try to group datapoints (e.i don't output the same cluster for all datapoints).
Hence the label mismatch fix can be applied for an extrinsic evaluation by mapping clusters containing a majority of preditions of a label to that label.
In [144]:
import numpy as np
def fix_clustering_label_mismatch(true_labels, predicted_labels):
""" Returns the predictions with the proper labels, a cluster represents the label most present in it."""
swap = False
# Change shape from (n_samples,) to (n_samples, n_classes)
labels = label_binarize(true_labels, classes=np.unique(true_labels))
predictions = label_binarize(predicted_labels, classes=np.unique(predicted_labels))
# Check if there is a label mismatch (below 50% accuracy)
if accuracy_score(labels, predictions) < 0.5:
# Swapping label encoding to match majority in clusters
fixed_predicted_labels = 1 - predicted_labels
swap = True
return true_labels, fixed_predicted_labels, swap
return true_labels, predicted_labels, swap
In [145]:
from sklearn.metrics import mutual_info_score
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import silhouette_score
n_labels = np.argmax(labels_generic_events, axis=-1)
swap_list = {}
swap_list["KMeans"] = []
swap_list["Agglomerative"] = []
clustering_ex_results_KMeans = {"features":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
clustering_ex_results_Agglomerative = {"features":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
clustering_in_results_KMeans = {"features":[], "mif":[], "ars":[], "silhouette":[], "MSE":[]}
clustering_in_results_Agglomerative = {"features":[], "mif":[], "ars":[], "silhouette":[], "MSE":[]}
for data_type in predictions:
print(f"-- Results for {data_type} input data --")
for algorithm in predictions[data_type][0]:
print(f"\n- Algorithm {algorithm}")
true_labels = n_labels
predicted_labels = predictions[data_type][0][algorithm][0]
# Assign label to cluster where it's more present
true_labels, predicted_labels, swap = fix_clustering_label_mismatch(true_labels, predicted_labels)
# To visualize the correct labels in plot
if "PCA" in data_type:
swap_list[algorithm].append(swap)
acc, auroc, auprc = get_metrics(
labels = true_labels,
predictions = predicted_labels
)
print("Extrinsic evaluation:")
print("-"*40)
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*30)
print("Intrinsic evaluation:")
print("-"*40)
mif = mutual_info_score(true_labels, predicted_labels)
print(f"Mutual info score: {mif}")
ars = adjusted_rand_score(true_labels, predicted_labels)
print(f"Adjusted rand score: {ars}")
silhouette = silhouette_score(true_labels.reshape(-1, 1), predicted_labels)
print(f"Silhouette score: {silhouette}")
MSE = mean_squared_error(true_labels, predicted_labels)
print(f"Mean squared error: {MSE}")
if algorithm == "KMeans":
clustering_ex_results_KMeans["features"].append(data_type)
clustering_ex_results_KMeans["accuracy"].append(acc)
clustering_ex_results_KMeans["AUROC"].append(auroc)
clustering_ex_results_KMeans["AUPRC"].append(auprc)
clustering_in_results_KMeans["features"].append(data_type)
clustering_in_results_KMeans["mif"].append(mif)
clustering_in_results_KMeans["ars"].append(ars)
clustering_in_results_KMeans["silhouette"].append(silhouette)
clustering_in_results_KMeans["MSE"].append(MSE)
elif algorithm == "Agglomerative":
clustering_ex_results_Agglomerative["features"].append({data_type})
clustering_ex_results_Agglomerative["accuracy"].append(acc)
clustering_ex_results_Agglomerative["AUROC"].append(auroc)
clustering_ex_results_Agglomerative["AUPRC"].append(auprc)
clustering_in_results_Agglomerative["features"].append({data_type})
clustering_in_results_Agglomerative["mif"].append(mif)
clustering_in_results_Agglomerative["ars"].append(ars)
clustering_in_results_Agglomerative["silhouette"].append(silhouette)
clustering_in_results_Agglomerative["MSE"].append(MSE)
print("-"*60)
print("")
-- Results for Raw input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.595 ------------------------------ AUROC: 0.596845238095238 ------------------------------ AUPRC: 0.5193937701662944 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.01880166161812219 Adjusted rand score: 0.035725338741475494 Silhouette score: 0.06044244678260578 Mean squared error: 0.405 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6 ------------------------------ AUROC: 0.5920238095238095 ------------------------------ AUPRC: 0.5189239383804601 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.01775557547950879 Adjusted rand score: 0.03937162945708509 Silhouette score: 0.06613334601984541 Mean squared error: 0.4 ------------------------------------------------------------ -- Results for PCA_raw input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5953846153846154 ------------------------------ AUROC: 0.5970833333333333 ------------------------------ AUPRC: 0.5195964604652945 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.018888676038459007 Adjusted rand score: 0.036019631076404844 Silhouette score: 0.0608944906765175 Mean squared error: 0.4046153846153846 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6307692307692307 ------------------------------ AUROC: 0.6339880952380952 ------------------------------ AUPRC: 0.5462151192994174 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.03636373290437567 Adjusted rand score: 0.06802537617769067 Silhouette score: 0.10856441587049105 Mean squared error: 0.36923076923076925 ------------------------------------------------------------ -- Results for Raw_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6646153846153846 ------------------------------ AUROC: 0.6766071428571427 ------------------------------ AUPRC: 0.5753121833256565 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.0707387970876742 Adjusted rand score: 0.10765151135637231 Silhouette score: 0.16352301788164328 Mean squared error: 0.3353846153846154 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.571923076923077 ------------------------------ AUROC: 0.5653571428571429 ------------------------------ AUPRC: 0.49960563380281686 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.008787082874513796 Adjusted rand score: 0.020139715578809718 Silhouette score: 0.03554749848556304 Mean squared error: 0.4280769230769231 ------------------------------------------------------------ -- Results for PCA_raw_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6592307692307692 ------------------------------ AUROC: 0.6707142857142856 ------------------------------ AUPRC: 0.5709124732583015 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.06529441580446627 Adjusted rand score: 0.10070470846049773 Silhouette score: 0.15420609019120057 Mean squared error: 0.34076923076923077 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6788461538461539 ------------------------------ AUROC: 0.6870833333333334 ------------------------------ AUPRC: 0.5861361669911314 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.07491400089767045 Adjusted rand score: 0.1274440748179099 Silhouette score: 0.1897416797960277 Mean squared error: 0.3211538461538462 ------------------------------------------------------------ -- Results for Features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6384615384615384 ------------------------------ AUROC: 0.6203571428571428 ------------------------------ AUPRC: 0.5517226598702503 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.0383717160877346 Adjusted rand score: 0.07510742071154326 Silhouette score: 0.11883614305999482 Mean squared error: 0.36153846153846153 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6480769230769231 ------------------------------ AUROC: 0.6279761904761905 ------------------------------ AUPRC: 0.5634561516914458 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.04718687865584606 Adjusted rand score: 0.085869496327039 Silhouette score: 0.13254317508013239 Mean squared error: 0.35192307692307695 ------------------------------------------------------------ -- Results for Features+Pooling input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6280769230769231 ------------------------------ AUROC: 0.6116666666666667 ------------------------------ AUPRC: 0.5410440827038338 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.031122806870876513 Adjusted rand score: 0.06423658745430076 Silhouette score: 0.10376452389574349 Mean squared error: 0.3719230769230769 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6476923076923077 ------------------------------ AUROC: 0.6278571428571429 ------------------------------ AUPRC: 0.5627559857361182 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.046556339766477106 Adjusted rand score: 0.08545139153816873 Silhouette score: 0.13216906806865963 Mean squared error: 0.3523076923076923 ------------------------------------------------------------ -- Results for Features_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.865 ------------------------------ AUROC: 0.8588095238095238 ------------------------------ AUPRC: 0.8157162628016407 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.30217324637488874 Adjusted rand score: 0.5326003592765147 Silhouette score: 0.617289254048675 Mean squared error: 0.135 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8496153846153847 ------------------------------ AUROC: 0.8561904761904762 ------------------------------ AUPRC: 0.760267896587676 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.29384580410601846 Adjusted rand score: 0.4886899186838671 Silhouette score: 0.580269317468822 Mean squared error: 0.1503846153846154 ------------------------------------------------------------ -- Results for Features+Pooling_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8938461538461538 ------------------------------ AUROC: 0.8915476190476191 ------------------------------ AUPRC: 0.8426602564102564 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.3522049451977292 Adjusted rand score: 0.6202836720432763 Silhouette score: 0.6941226729781622 Mean squared error: 0.10615384615384615 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8688461538461538 ------------------------------ AUROC: 0.8711309523809524 ------------------------------ AUPRC: 0.7931221343251275 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.30848743229302944 Adjusted rand score: 0.5440155194055404 Silhouette score: 0.6275085903607341 Mean squared error: 0.13115384615384615 ------------------------------------------------------------ -- Results for PCA_features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6411538461538462 ------------------------------ AUROC: 0.6235714285714284 ------------------------------ AUPRC: 0.554164689935615 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.03979640282204272 Adjusted rand score: 0.07818833106788244 Silhouette score: 0.12317512109095588 Mean squared error: 0.35884615384615387 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6488461538461539 ------------------------------ AUROC: 0.6299404761904762 ------------------------------ AUPRC: 0.5631297193620028 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.04636562937912242 Adjusted rand score: 0.08694913907617303 Silhouette score: 0.1346832120122245 Mean squared error: 0.35115384615384615 ------------------------------------------------------------ -- Results for PCA_features_pooling input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.6303846153846154 ------------------------------ AUROC: 0.6151785714285715 ------------------------------ AUPRC: 0.5429844238330477 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.03205224661615563 Adjusted rand score: 0.06676229538776672 Silhouette score: 0.10729766349124768 Mean squared error: 0.3696153846153846 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5834615384615385 ------------------------------ AUROC: 0.5777380952380953 ------------------------------ AUPRC: 0.5078864368140262 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.01235832802781961 Adjusted rand score: 0.027352148304607197 Silhouette score: 0.047204080735378684 Mean squared error: 0.4165384615384615 ------------------------------------------------------------ -- Results for PCA_features_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8576923076923076 ------------------------------ AUROC: 0.8517857142857143 ------------------------------ AUPRC: 0.8036034353995519 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.2867895241234161 Adjusted rand score: 0.5114716590539701 Silhouette score: 0.5975968325478581 Mean squared error: 0.1423076923076923 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8811538461538462 ------------------------------ AUROC: 0.8780952380952382 ------------------------------ AUPRC: 0.8269466364738444 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.3265340319183105 Adjusted rand score: 0.5809087445091459 Silhouette score: 0.6595567119005019 Mean squared error: 0.11884615384615385 ------------------------------------------------------------ -- Results for PCA_features_pooling_rgb input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.8688461538461538 ------------------------------ AUROC: 0.8651785714285712 ------------------------------ AUPRC: 0.8113091917149033 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.3029963063488798 Adjusted rand score: 0.5439575487216932 Silhouette score: 0.6264752053285271 Mean squared error: 0.13115384615384615 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.7834615384615384 ------------------------------ AUROC: 0.7687499999999999 ------------------------------ AUPRC: 0.7293224299065422 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.19564189719120412 Adjusted rand score: 0.3204662160538807 Silhouette score: 0.41276437527141246 Mean squared error: 0.21653846153846154 ------------------------------------------------------------ -- Results for Matrix_features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5392307692307692 ------------------------------ AUROC: 0.5241666666666667 ------------------------------ AUPRC: 0.4745843935538592 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.0013742157295733026 Adjusted rand score: 0.004914628538723195 Silhouette score: 0.009990589102050728 Mean squared error: 0.46076923076923076 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5388461538461539 ------------------------------ AUROC: 0.5089285714285715 ------------------------------ AUPRC: 0.4663629053872957 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.00040243027451011826 Adjusted rand score: 0.002301566322516027 Silhouette score: 0.008719329356417064 Mean squared error: 0.46115384615384614 ------------------------------------------------------------ -- Results for PCA_matrix_features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5392307692307692 ------------------------------ AUROC: 0.5241666666666667 ------------------------------ AUPRC: 0.4745843935538592 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.0013742157295733026 Adjusted rand score: 0.004914628538723195 Silhouette score: 0.009990589102050728 Mean squared error: 0.46076923076923076 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5388461538461539 ------------------------------ AUROC: 0.5089285714285715 ------------------------------ AUPRC: 0.4663629053872957 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.00040243027451011826 Adjusted rand score: 0.002301566322516027 Silhouette score: 0.008719329356417064 Mean squared error: 0.46115384615384614 ------------------------------------------------------------ -- Results for Hog_features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5611538461538461 ------------------------------ AUROC: 0.5808928571428571 ------------------------------ AUPRC: 0.5064133777549974 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.017823099885256904 Adjusted rand score: 0.013212146934626497 Silhouette score: 0.016259148150338222 Mean squared error: 0.43884615384615383 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5634615384615385 ------------------------------ AUROC: 0.5939880952380953 ------------------------------ AUPRC: 0.5135696509140089 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.05595494473788535 Adjusted rand score: 0.01241278685242793 Silhouette score: -0.07785522816668002 Mean squared error: 0.43653846153846154 ------------------------------------------------------------ -- Results for PCA_hog_features input data -- - Algorithm KMeans Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5665384615384615 ------------------------------ AUROC: 0.5838095238095238 ------------------------------ AUPRC: 0.5084311867386319 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.017619918617434116 Adjusted rand score: 0.016297511554227886 Silhouette score: 0.02483166413958898 Mean squared error: 0.43346153846153845 - Algorithm Agglomerative Extrinsic evaluation: ---------------------------------------- Accuracy: 0.5642307692307692 ------------------------------ AUROC: 0.5720833333333333 ------------------------------ AUPRC: 0.5020292532526576 ------------------------------ Intrinsic evaluation: ---------------------------------------- Mutual info score: 0.010809198635083173 Adjusted rand score: 0.01592293040512831 Silhouette score: 0.02836691063325809 Mean squared error: 0.43576923076923074 ------------------------------------------------------------
In [146]:
# 2D Plots
labels = np.unique(n_labels)
for algorithm in ("KMeans", "Agglomerative"):
for data_type, decomposed_data, clustering_predictions, swap in (
("PCA_raw_input", PCA_raw_input, predictions["PCA_raw"][0][algorithm][0], swap_list[algorithm][0]),
("PCA_raw_input_rgb", PCA_raw_input_rgb, predictions["PCA_raw_rgb"][0][algorithm][0], swap_list[algorithm][1]),
("PCA_features", PCA_features, predictions["PCA_features"][0][algorithm][0], swap_list[algorithm][2]),
("PCA_features_pooling", PCA_features_pooling, predictions["PCA_features_pooling"][0][algorithm][0], swap_list[algorithm][3]),
("PCA_features_rgb", PCA_features_rgb, predictions["PCA_features_rgb"][0][algorithm][0], swap_list[algorithm][4]),
("PCA_features_pooling_rgb", PCA_features_pooling_rgb, predictions["PCA_features_pooling_rgb"][0][algorithm][0], swap_list[algorithm][5]),
("PCA_matrix_features", PCA_matrix_features, predictions["PCA_matrix_features"][0][algorithm][0], swap_list[algorithm][6]),
("PCA_hog_features", PCA_hog_features, predictions["PCA_hog_features"][0][algorithm][0], swap_list[algorithm][7])):
plt.figure(figsize=(12,6))
for l in labels:
plt.scatter(decomposed_data[np.squeeze(n_labels == l), 0], decomposed_data[np.squeeze(n_labels == l), 1], label = l, marker = ".")
plt.show()
print(f"- {algorithm} predictions on {data_type} -")
if swap:
plt.figure(figsize=(12,6))
for l in labels:
plt.scatter(decomposed_data[clustering_predictions == 1-l, 0], decomposed_data[clustering_predictions == 1-l, 1], label = l, marker =".")
plt.show()
else:
plt.figure(figsize=(12,6))
for l in labels:
plt.scatter(decomposed_data[clustering_predictions == l, 0], decomposed_data[clustering_predictions == l, 1], label = l, marker =".")
plt.show()
print("-"*100)
- KMeans predictions on PCA_raw_input -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_raw_input_rgb -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_features -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_features_pooling -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_features_rgb -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_features_pooling_rgb -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_matrix_features -
----------------------------------------------------------------------------------------------------
- KMeans predictions on PCA_hog_features -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_raw_input -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_raw_input_rgb -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_features -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_features_pooling -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_features_rgb -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_features_pooling_rgb -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_matrix_features -
----------------------------------------------------------------------------------------------------
- Agglomerative predictions on PCA_hog_features -
----------------------------------------------------------------------------------------------------
In [147]:
results_ex_clusteringDF_KMeans = pd.DataFrame(clustering_ex_results_KMeans)
results_ex_clusteringDF_Agglomerative = pd.DataFrame(clustering_ex_results_Agglomerative)
results_in_clusteringDF_KMeans = pd.DataFrame(clustering_in_results_KMeans)
results_in_clusteringDF_Agglomerative = pd.DataFrame(clustering_in_results_Agglomerative)
In [148]:
from barplots import barplots
barplots(
results_ex_clusteringDF_KMeans,
groupby=["features"],
orientation="horizontal",
height=12,
legend_position="upper left"
)
Out[148]:
[(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object))]
In [149]:
from barplots import barplots
barplots(
results_ex_clusteringDF_Agglomerative,
groupby=["features"],
orientation="horizontal",
height=12,
legend_position="upper left"
)
Out[149]:
[(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object))]
In [150]:
from barplots import barplots
barplots(
results_in_clusteringDF_KMeans,
groupby=["features"],
orientation="horizontal",
height=12,
legend_position="upper left"
)
Out[150]:
[(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'MSE'}, xlabel='MSE'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Silhouette'}, xlabel='Silhouette'>],
dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Ars'}, xlabel='Ars'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Mif'}, xlabel='Mif'>], dtype=object))]
In [151]:
from barplots import barplots
barplots(
results_in_clusteringDF_Agglomerative,
groupby=["features"],
orientation="horizontal",
height=12,
legend_position="upper left"
)
Out[151]:
[(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'MSE'}, xlabel='MSE'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Silhouette'}, xlabel='Silhouette'>],
dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Ars'}, xlabel='Ars'>], dtype=object)),
(<Figure size 2400x960 with 1 Axes>,
array([<Axes: title={'center': 'Mif'}, xlabel='Mif'>], dtype=object))]
